sync run artifacts: vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6bc222a7d5b17804aa9cd2e2d761ac4415cea396125d9456d3f2c7e877b0ffd
+size 6191381293

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank0.log

@@ -0,0 +1,28 @@
+Rank 0 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank1.log

@@ -0,0 +1,28 @@
+Rank 1 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank10.log

@@ -0,0 +1,28 @@
+Rank 10 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank11.log

@@ -0,0 +1,28 @@
+Rank 11 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank12.log

@@ -0,0 +1,28 @@
+Rank 12 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank13.log

@@ -0,0 +1,28 @@
+Rank 13 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank14.log

@@ -0,0 +1,28 @@
+Rank 14 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank15.log

@@ -0,0 +1,28 @@
+Rank 15 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank2.log

@@ -0,0 +1,28 @@
+Rank 2 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank3.log

@@ -0,0 +1,28 @@
+Rank 3 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank4.log

@@ -0,0 +1,28 @@
+Rank 4 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank5.log

@@ -0,0 +1,28 @@
+Rank 5 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank6.log

@@ -0,0 +1,28 @@
+Rank 6 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank7.log

@@ -0,0 +1,28 @@
+Rank 7 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank8.log

@@ -0,0 +1,28 @@
+Rank 8 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank9.log

@@ -0,0 +1,28 @@
+Rank 9 crashed at epoch 11, itr 59
+Error: loss is nan
+
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/git-info.txt

@@ -0,0 +1,2 @@
+branch: main
+commit: 4f75caae343937a59636f9600dd815d5d6b06a6f

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_0_log.err

@@ -0,0 +1,73 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  return func(*args, **kwargs)
+[rank0]:[W510 10:50:40.536263067 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+wandb: Currently logged in as: dgcnz (uvjepa) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 0ebazd1x
+wandb: Tracking run with wandb version 0.23.1
+wandb: Run data is saved locally in /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/wandb/run-20260510_105047-0ebazd1x
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run c001_vitl_k32_simple_cross_stable
+wandb: ⭐️ View project at https://wandb.ai/uvjepa/vjepa_ujepaside
+wandb: 🚀 View run at https://wandb.ai/uvjepa/vjepa_ujepaside/runs/0ebazd1x
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  return func(*args, **kwargs)
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  return func(*args, **kwargs)
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  return func(*args, **kwargs)
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,936) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank0]:[W510 14:06:45.675209065 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+srun: error: gcn75: task 5: Exited with exit code 1
+srun: Terminating StepId=22613067.0
+[2026-05-10T14:06:45.953] error: *** STEP 22613067.0 ON gcn74 CANCELLED AT 2026-05-10T14:06:45 DUE TO TASK FAILURE ***
+srun: error: gcn77: task 12: Exited with exit code 1
+srun: error: gcn75: task 6: Exited with exit code 1
+srun: error: gcn77: task 14: Exited with exit code 1
+srun: error: gcn74: task 3: Exited with exit code 1
+srun: error: gcn76: task 11: Terminated
+srun: error: gcn75: task 7: Exited with exit code 1
+srun: error: gcn76: task 9: Terminated
+srun: error: gcn77: task 15: Exited with exit code 1
+srun: error: gcn74: task 1: Exited with exit code 1
+srun: error: gcn75: task 4: Terminated
+srun: error: gcn76: task 8: Terminated
+srun: error: gcn77: task 13: Exited with exit code 1
+srun: error: gcn76: task 10: Terminated
+srun: error: gcn74: task 2: Terminated
+srun: error: gcn74: task 0: Terminated
+srun: Force Terminated StepId=22613067.0

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_10_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank10]:[W510 10:49:36.780200773 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,946) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank10]:[W510 14:06:44.843889483 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_10_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn76.local.snellius.surf.nl, local_rank=2(4), node=2(4), global_rank=10(16))
+submitit INFO (2026-05-10 10:45:50,696) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:33][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 10/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:33][root                ][stage_datasets           ] [local_rank 2/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:33][root                ][_stage_targz_parts       ] [rank 2] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:47][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:47:00][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:15][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:29][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:43][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:56][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:10][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:25][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:39][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:53][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:08][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:22][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:36][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:36][root                ][stage_datasets           ] [local_rank 2/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:11][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:23][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:28][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 10 / 16
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 10)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,946) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank10.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_11_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank11]:[W510 10:49:32.803633407 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:43,177) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank11]:[W510 14:06:44.814354209 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_11_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn76.local.snellius.surf.nl, local_rank=3(4), node=2(4), global_rank=11(16))
+submitit INFO (2026-05-10 10:45:50,696) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:34][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 11/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:34][root                ][stage_datasets           ] [local_rank 3/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:34][root                ][_stage_targz_parts       ] [rank 3] Extracting 25/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:48][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 2/25 parts
+[INFO    ][2026-05-10 10:47:02][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 4/25 parts
+[INFO    ][2026-05-10 10:47:16][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 6/25 parts
+[INFO    ][2026-05-10 10:47:30][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 8/25 parts
+[INFO    ][2026-05-10 10:47:45][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 10/25 parts
+[INFO    ][2026-05-10 10:47:59][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 12/25 parts
+[INFO    ][2026-05-10 10:48:13][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 14/25 parts
+[INFO    ][2026-05-10 10:48:28][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 16/25 parts
+[INFO    ][2026-05-10 10:48:42][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 18/25 parts
+[INFO    ][2026-05-10 10:48:56][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 20/25 parts
+[INFO    ][2026-05-10 10:49:10][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 22/25 parts
+[INFO    ][2026-05-10 10:49:24][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 24/25 parts
+[INFO    ][2026-05-10 10:49:32][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 25/25 parts
+[INFO    ][2026-05-10 10:49:32][root                ][stage_datasets           ] [local_rank 3/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:11][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:23][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:28][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 11 / 16
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 11)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:43,177) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank11.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_12_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank12]:[W510 10:50:31.359507529 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,226) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank12]:[W510 14:06:42.186233905 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_12_log.out

@@ -0,0 +1,280 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn77.local.snellius.surf.nl, local_rank=0(4), node=3(4), global_rank=12(16))
+submitit INFO (2026-05-10 10:45:50,697) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:35][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 12/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:35][root                ][stage_datasets           ] [local_rank 0/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:35][root                ][_stage_targz_parts       ] [rank 0] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:49][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:47:03][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:16][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:30][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:44][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:59][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:13][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:27][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:41][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:55][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:09][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:24][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:38][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:38][root                ][stage_datasets           ] [local_rank 0/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:49:38][root                ][_stage_multipart_tar     ] [rank 0] Extracting multipart tar (2 files) to /scratch-node/dcanez.22613067/ssv2
+[INFO    ][2026-05-10 10:50:43][root                ][stage_datasets           ] Data staging completed in 248.4s (4.1min)
+[INFO    ][2026-05-10 10:50:43][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22613067/kinetics_240/train.csv (239789 entries)
+[INFO    ][2026-05-10 10:50:44][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22613067/ssv2/train.csv (168913 entries)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:11][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:23][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:28][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 12 / 16
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 12)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,226) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:42][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank12.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_13_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank13]:[W510 10:49:35.472727121 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:43,172) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank13]:[W510 14:06:44.707111075 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_13_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn77.local.snellius.surf.nl, local_rank=1(4), node=3(4), global_rank=13(16))
+submitit INFO (2026-05-10 10:45:50,696) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:33][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 13/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:33][root                ][stage_datasets           ] [local_rank 1/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:33][root                ][_stage_targz_parts       ] [rank 1] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:46][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:46:59][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:14][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:27][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:42][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:56][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:09][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:23][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:37][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:51][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:06][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:20][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:35][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:35][root                ][stage_datasets           ] [local_rank 1/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:11][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:23][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:28][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 13 / 16
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 13)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:43,172) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank13.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_14_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank14]:[W510 10:49:36.494686462 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,937) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank14]:[W510 14:06:44.703786579 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_14_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn77.local.snellius.surf.nl, local_rank=2(4), node=3(4), global_rank=14(16))
+submitit INFO (2026-05-10 10:45:50,697) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:33][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 14/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:33][root                ][stage_datasets           ] [local_rank 2/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:33][root                ][_stage_targz_parts       ] [rank 2] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:46][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:47:00][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:14][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:29][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:43][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:57][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:11][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:25][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:39][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:53][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:07][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:21][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:36][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:36][root                ][stage_datasets           ] [local_rank 2/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:11][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:23][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:28][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 14 / 16
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 14)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,937) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank14.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_15_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank15]:[W510 10:49:32.526269737 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,946) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank15]:[W510 14:06:44.721708541 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_15_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn77.local.snellius.surf.nl, local_rank=3(4), node=3(4), global_rank=15(16))
+submitit INFO (2026-05-10 10:45:50,697) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:34][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 15/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:34][root                ][stage_datasets           ] [local_rank 3/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:34][root                ][_stage_targz_parts       ] [rank 3] Extracting 25/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:48][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 2/25 parts
+[INFO    ][2026-05-10 10:47:02][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 4/25 parts
+[INFO    ][2026-05-10 10:47:16][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 6/25 parts
+[INFO    ][2026-05-10 10:47:30][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 8/25 parts
+[INFO    ][2026-05-10 10:47:45][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 10/25 parts
+[INFO    ][2026-05-10 10:47:59][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 12/25 parts
+[INFO    ][2026-05-10 10:48:13][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 14/25 parts
+[INFO    ][2026-05-10 10:48:28][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 16/25 parts
+[INFO    ][2026-05-10 10:48:41][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 18/25 parts
+[INFO    ][2026-05-10 10:48:56][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 20/25 parts
+[INFO    ][2026-05-10 10:49:10][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 22/25 parts
+[INFO    ][2026-05-10 10:49:24][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 24/25 parts
+[INFO    ][2026-05-10 10:49:32][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 25/25 parts
+[INFO    ][2026-05-10 10:49:32][root                ][stage_datasets           ] [local_rank 3/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:11][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:23][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:28][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 15 / 16
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 15)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,946) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank15.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_1_log.err

@@ -0,0 +1,38 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank1]:[W510 10:49:35.449735542 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,945) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank1]:[W510 14:06:44.592867363 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_1_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn74.local.snellius.surf.nl, local_rank=1(4), node=0(4), global_rank=1(16))
+submitit INFO (2026-05-10 10:45:50,696) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:31][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:31][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:31][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:32][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 1/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:32][root                ][stage_datasets           ] [local_rank 1/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:32][root                ][_stage_targz_parts       ] [rank 1] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:45][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:46:59][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:13][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:27][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:41][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:55][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:09][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:23][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:37][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:51][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:06][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:20][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:35][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:35][root                ][stage_datasets           ] [local_rank 1/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:52][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:58][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:52:01][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:52:01][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 1 / 16
+[INFO    ][2026-05-10 10:52:01][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 1)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,945) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank1.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_2_log.err

@@ -0,0 +1,38 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank2]:[W510 10:49:36.441527182 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:43,171) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank2]:[W510 14:06:44.773812020 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_2_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn74.local.snellius.surf.nl, local_rank=2(4), node=0(4), global_rank=2(16))
+submitit INFO (2026-05-10 10:45:50,696) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:31][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:31][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:31][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:33][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 2/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:33][root                ][stage_datasets           ] [local_rank 2/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:33][root                ][_stage_targz_parts       ] [rank 2] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:47][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:47:00][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:15][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:29][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:43][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:57][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:11][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:25][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:39][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:53][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:08][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:22][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:36][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:36][root                ][stage_datasets           ] [local_rank 2/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:52][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:58][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:59][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:59][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:59][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:59][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:52:01][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:52:01][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 2 / 16
+[INFO    ][2026-05-10 10:52:01][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 2)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:43,171) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank2.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_3_log.err

@@ -0,0 +1,38 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank3]:[W510 10:49:32.556389443 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,226) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank3]:[W510 14:06:42.736826829 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_3_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn74.local.snellius.surf.nl, local_rank=3(4), node=0(4), global_rank=3(16))
+submitit INFO (2026-05-10 10:45:50,696) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:31][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:31][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:31][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:33][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 3/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:33][root                ][stage_datasets           ] [local_rank 3/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:33][root                ][_stage_targz_parts       ] [rank 3] Extracting 25/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:47][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 2/25 parts
+[INFO    ][2026-05-10 10:47:02][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 4/25 parts
+[INFO    ][2026-05-10 10:47:16][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 6/25 parts
+[INFO    ][2026-05-10 10:47:30][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 8/25 parts
+[INFO    ][2026-05-10 10:47:45][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 10/25 parts
+[INFO    ][2026-05-10 10:47:59][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 12/25 parts
+[INFO    ][2026-05-10 10:48:13][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 14/25 parts
+[INFO    ][2026-05-10 10:48:28][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 16/25 parts
+[INFO    ][2026-05-10 10:48:41][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 18/25 parts
+[INFO    ][2026-05-10 10:48:56][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 20/25 parts
+[INFO    ][2026-05-10 10:49:10][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 22/25 parts
+[INFO    ][2026-05-10 10:49:24][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 24/25 parts
+[INFO    ][2026-05-10 10:49:32][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 25/25 parts
+[INFO    ][2026-05-10 10:49:32][root                ][stage_datasets           ] [local_rank 3/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:52][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:58][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:58][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:52:01][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:52:01][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 3 / 16
+[INFO    ][2026-05-10 10:52:01][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 3)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,226) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank3.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_4_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank4]:[W510 10:50:42.458578063 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,946) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank4]:[W510 14:06:44.681088353 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_4_log.out

@@ -0,0 +1,280 @@
+submitit INFO (2026-05-10 10:45:47,883) - Starting with JobEnvironment(job_id=22613067, hostname=gcn75.local.snellius.surf.nl, local_rank=0(4), node=1(4), global_rank=4(16))
+submitit INFO (2026-05-10 10:45:47,883) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:43][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 4/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:43][root                ][stage_datasets           ] [local_rank 0/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:43][root                ][_stage_targz_parts       ] [rank 0] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:57][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:47:11][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:24][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:39][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:53][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:48:07][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:21][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:35][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:49][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:49:03][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:17][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:31][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:45][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:45][root                ][stage_datasets           ] [local_rank 0/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:49:45][root                ][_stage_multipart_tar     ] [rank 0] Extracting multipart tar (2 files) to /scratch-node/dcanez.22613067/ssv2
+[INFO    ][2026-05-10 10:50:43][root                ][stage_datasets           ] Data staging completed in 240.4s (4.0min)
+[INFO    ][2026-05-10 10:50:43][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22613067/kinetics_240/train.csv (239789 entries)
+[INFO    ][2026-05-10 10:50:44][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22613067/ssv2/train.csv (168913 entries)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:10][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:21][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:22][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:22][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 4 / 16
+[INFO    ][2026-05-10 10:51:22][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:22][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:22][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 4)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,946) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank4.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_5_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank5]:[W510 10:49:36.789639922 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,226) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank5]:[W510 14:06:42.480093455 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_5_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:47,883) - Starting with JobEnvironment(job_id=22613067, hostname=gcn75.local.snellius.surf.nl, local_rank=1(4), node=1(4), global_rank=5(16))
+submitit INFO (2026-05-10 10:45:47,883) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:34][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 5/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:34][root                ][stage_datasets           ] [local_rank 1/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:34][root                ][_stage_targz_parts       ] [rank 1] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:47][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:47:01][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:15][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:29][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:43][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:57][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:10][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:24][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:39][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:53][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:07][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:22][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:36][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:36][root                ][stage_datasets           ] [local_rank 1/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:10][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:21][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:22][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:22][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 5 / 16
+[INFO    ][2026-05-10 10:51:22][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:22][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:22][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 5)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,226) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank5.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_6_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank6]:[W510 10:49:36.414267391 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:43,183) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank6]:[W510 14:06:44.647457979 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_6_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:47,883) - Starting with JobEnvironment(job_id=22613067, hostname=gcn75.local.snellius.surf.nl, local_rank=2(4), node=1(4), global_rank=6(16))
+submitit INFO (2026-05-10 10:45:47,883) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:32][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 6/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:32][root                ][stage_datasets           ] [local_rank 2/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:32][root                ][_stage_targz_parts       ] [rank 2] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:46][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:47:00][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:14][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:29][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:43][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:57][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:10][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:25][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:39][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:53][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:08][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:22][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:35][root                ][_stage_targz_parts       ] [local_rank 2] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:35][root                ][stage_datasets           ] [local_rank 2/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:10][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:21][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:22][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:22][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 6 / 16
+[INFO    ][2026-05-10 10:51:22][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:22][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:22][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 6)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:43,183) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank6.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_7_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank7]:[W510 10:49:34.461031118 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,937) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank7]:[W510 14:06:43.548753326 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_7_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:47,883) - Starting with JobEnvironment(job_id=22613067, hostname=gcn75.local.snellius.surf.nl, local_rank=3(4), node=1(4), global_rank=7(16))
+submitit INFO (2026-05-10 10:45:47,883) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:08][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:36][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 7/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:36][root                ][stage_datasets           ] [local_rank 3/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:36][root                ][_stage_targz_parts       ] [rank 3] Extracting 25/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:50][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 2/25 parts
+[INFO    ][2026-05-10 10:47:05][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 4/25 parts
+[INFO    ][2026-05-10 10:47:19][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 6/25 parts
+[INFO    ][2026-05-10 10:47:33][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 8/25 parts
+[INFO    ][2026-05-10 10:47:48][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 10/25 parts
+[INFO    ][2026-05-10 10:48:02][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 12/25 parts
+[INFO    ][2026-05-10 10:48:16][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 14/25 parts
+[INFO    ][2026-05-10 10:48:30][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 16/25 parts
+[INFO    ][2026-05-10 10:48:44][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 18/25 parts
+[INFO    ][2026-05-10 10:48:58][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 20/25 parts
+[INFO    ][2026-05-10 10:49:13][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 22/25 parts
+[INFO    ][2026-05-10 10:49:26][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 24/25 parts
+[INFO    ][2026-05-10 10:49:34][root                ][_stage_targz_parts       ] [local_rank 3] Extracted 25/25 parts
+[INFO    ][2026-05-10 10:49:34][root                ][stage_datasets           ] [local_rank 3/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:10][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:21][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:21][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:22][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:22][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 7 / 16
+[INFO    ][2026-05-10 10:51:22][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:22][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:22][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 7)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,937) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:42][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank7.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_8_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank8]:[W510 10:50:31.090660717 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:43,172) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank8]:[W510 14:06:44.809824319 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_8_log.out

@@ -0,0 +1,280 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn76.local.snellius.surf.nl, local_rank=0(4), node=2(4), global_rank=8(16))
+submitit INFO (2026-05-10 10:45:50,696) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:32][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 8/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:32][root                ][stage_datasets           ] [local_rank 0/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:32][root                ][_stage_targz_parts       ] [rank 0] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:46][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:47:01][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:15][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:29][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:43][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:58][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:12][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:27][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:41][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:55][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:09][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:24][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:38][root                ][_stage_targz_parts       ] [local_rank 0] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:38][root                ][stage_datasets           ] [local_rank 0/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:49:38][root                ][_stage_multipart_tar     ] [rank 0] Extracting multipart tar (2 files) to /scratch-node/dcanez.22613067/ssv2
+[INFO    ][2026-05-10 10:50:43][root                ][stage_datasets           ] Data staging completed in 250.8s (4.2min)
+[INFO    ][2026-05-10 10:50:43][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22613067/kinetics_240/train.csv (239789 entries)
+[INFO    ][2026-05-10 10:50:44][root                ][_rewrite_csv             ] Wrote local CSV: /scratch-node/dcanez.22613067/ssv2/train.csv (168913 entries)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:11][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:23][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:28][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 8 / 16
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 8)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:43,172) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank8.log

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_9_log.err

@@ -0,0 +1,34 @@
+/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/timm/models/layers/__init__.py:49: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+[rank9]:[W510 10:49:36.464305364 ProcessGroupNCCL.cpp:5138] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/utils.py:796: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  scaler = torch.cuda.amp.GradScaler() if mixed_precision else None
+/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py:955: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype, enabled=mixed_precision):
+submitit ERROR (2026-05-10 14:06:41,937) - Submitted job triggered an exception
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/_submit.py", line 11, in <module>
+    submitit_main()
+    ~~~~~~~~~~~~~^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 76, in submitit_main
+    process_job(args.folder)
+    ~~~~~~~~~~~^^^^^^^^^^^^^
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 69, in process_job
+    raise error
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/submission.py", line 55, in process_job
+    result = delayed.result()
+  File "/scratch-shared/dcanez/.cache/uv5/virtualenvs/vd/lib/python3.13/site-packages/submitit/core/utils.py", line 137, in result
+    self._result = self.function(*self.args, **self.kwargs)
+                   ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/dcanez/vd/app/main_distributed.py", line 97, in __call__
+    pp = pprint.PrettyPrinter(indent=4)
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/scaffold.py", line 17, in main
+    return importlib.import_module(f"app.{app}.train").main(args=args, resume_preempt=resume_preempt)
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/scratch1/shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/code/app/vjepa/train.py", line 1189, in main
+    assert not np.isnan(loss), "loss is nan"
+           ^^^^^^^^^^^^^^^^^^
+AssertionError: loss is nan
+[rank9]:[W510 14:06:44.834298574 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_9_log.out

@@ -0,0 +1,276 @@
+submitit INFO (2026-05-10 10:45:50,696) - Starting with JobEnvironment(job_id=22613067, hostname=gcn76.local.snellius.surf.nl, local_rank=1(4), node=2(4), global_rank=9(16))
+submitit INFO (2026-05-10 10:45:50,696) - Loading pickle: /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/job_22613067/22613067_submitted.pkl
+INFO:root:loaded pretrain params...
+{   'app': 'vjepa',
+    'cpus_per_task': 16,
+    'data': {   'batch_size': 64,
+                'crop_size': 224,
+                'dataset_fpcs': [16, 16],
+                'dataset_type': 'VideoDataset',
+                'datasets': [   '/scratch-shared/dcanez/data/kinetics/k400/train.csv',
+                                '/scratch-shared/dcanez/data/ssv2/train.csv'],
+                'datasets_weights': [0.65, 0.35],
+                'fps': 4,
+                'num_workers': 10,
+                'patch_size': 14,
+                'persistent_workers': True,
+                'pin_mem': True,
+                'stage': [   {   'dest': 'kinetics_240',
+                                 'format': 'targz_parts',
+                                 'src': '/scratch-shared/dcanez/data/kinetics/k400/tars_240/'},
+                             {   'dest': 'ssv2',
+                                 'format': 'multipart_tar',
+                                 'src': '/scratch-nvme/ml-datasets/something-something-v2/'}],
+                'tubelet_size': 1},
+    'data_aug': {   'auto_augment': False,
+                    'motion_shift': False,
+                    'random_resize_aspect_ratio': [0.75, 1.35],
+                    'random_resize_scale': [0.3, 1.0],
+                    'reprob': 0.0},
+    'folder': '/scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable',
+    'loss': {'loss_exp': 1.0},
+    'mask': [   {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 8,
+                    'spatial_scale': [0.15, 0.15],
+                    'temporal_scale': [1.0, 1.0]},
+                {   'aspect_ratio': [0.75, 1.5],
+                    'full_complement': False,
+                    'max_keep': None,
+                    'max_temporal_keep': 1.0,
+                    'num_blocks': 2,
+                    'spatial_scale': [0.7, 0.7],
+                    'temporal_scale': [1.0, 1.0]}],
+    'mem_per_gpu': '180G',
+    'meta': {   'dtype': 'bfloat16',
+                'knn_eval_epoch0': False,
+                'knn_eval_freq': 5,
+                'knn_eval_presets': [   {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/train.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/ucf101/ucfTrainTestlist/val.csv',
+                                                          'eval_videos_per_class': 25,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 100},
+                                            'preset': 'ucf101'},
+                                        {   'config': {   'batch_size': 64,
+                                                          'dataset_train': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/train_coarse10.csv',
+                                                          'dataset_val': '/scratch-shared/mdorkenw/20bn-something-something-v2/something-something-v2-annotations/val_coarse10.csv',
+                                                          'eval_videos_per_class': 100,
+                                                          'linear_probe': True,
+                                                          'num_workers': 8,
+                                                          'pool_type': 'slot_temporal_concat',
+                                                          'train_videos_per_class': 500},
+                                            'preset': 'ssv2_coarse10'}],
+                'load_checkpoint': True,
+                'read_checkpoint': None,
+                'save_every_freq': 5,
+                'seed': 239,
+                'use_sdpa': True,
+                'use_wandb': True,
+                'wandb_project': 'vjepa_ujepaside'},
+    'metrics': {'sigreg': {}, 'std': {}},
+    'model': {   'model_name': 'ujepaside_large_patch14_capi_lvd1689m',
+                 'pred_depth': 6,
+                 'pred_embed_dim': 384,
+                 'pred_num_heads': 12,
+                 'predictor': 'v2_cross',
+                 'st_causal': False,
+                 'st_drop_path': 0.2,
+                 'st_flex_enable': False,
+                 'st_layer_scale_init': 1e-05,
+                 'st_num_slots': 32,
+                 'st_slots_causal_within_frame': False,
+                 'target_kind': 'frozen_2d',
+                 'target_type': 'vit_large_patch14_capi.lvd1689m',
+                 'temporal_spacing': 1.0,
+                 'uniform_power': True,
+                 'use_activation_checkpointing': True,
+                 'use_mask_tokens': True,
+                 'use_rope': True,
+                 'use_sdpa': True,
+                 'zero_init_mask_tokens': True},
+    'nodes': 4,
+    'optimization': {   'clip_grad': 3.0,
+                        'ema': [0.99925, 0.99925],
+                        'epochs': 100,
+                        'final_lr': 0.0001,
+                        'final_weight_decay': 0.04,
+                        'ipe': 300,
+                        'ipe_scale': 1.0,
+                        'lr': 0.0005,
+                        'start_lr': 0.0001,
+                        'warmup': 10,
+                        'weight_decay': 0.04},
+    'tasks_per_node': 4}
+INFO:root:Running pre-training of app: vjepa
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] which_dtype='bfloat16'
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] Disabling persistent_workers (incompatible with KNN eval)
+[INFO    ][2026-05-10 10:46:30][app.vjepa.train     ][main                     ] NCCL_SOCKET_IFNAME=eno
+[INFO    ][2026-05-10 10:46:34][app.vjepa.train     ][main                     ] Initialized (rank/world-size) 9/16, tasks_per_node=4
+[INFO    ][2026-05-10 10:46:34][root                ][stage_datasets           ] [local_rank 1/4] Staging kinetics_240 (targz_parts)
+[INFO    ][2026-05-10 10:46:34][root                ][_stage_targz_parts       ] [rank 1] Extracting 26/103 tar.gz parts to /scratch-node/dcanez.22613067/kinetics_240
+[INFO    ][2026-05-10 10:46:47][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 2/26 parts
+[INFO    ][2026-05-10 10:47:00][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 4/26 parts
+[INFO    ][2026-05-10 10:47:14][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 6/26 parts
+[INFO    ][2026-05-10 10:47:28][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 8/26 parts
+[INFO    ][2026-05-10 10:47:43][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 10/26 parts
+[INFO    ][2026-05-10 10:47:57][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 12/26 parts
+[INFO    ][2026-05-10 10:48:10][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 14/26 parts
+[INFO    ][2026-05-10 10:48:24][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 16/26 parts
+[INFO    ][2026-05-10 10:48:38][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 18/26 parts
+[INFO    ][2026-05-10 10:48:53][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 20/26 parts
+[INFO    ][2026-05-10 10:49:07][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 22/26 parts
+[INFO    ][2026-05-10 10:49:21][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 24/26 parts
+[INFO    ][2026-05-10 10:49:35][root                ][_stage_targz_parts       ] [local_rank 1] Extracted 26/26 parts
+[INFO    ][2026-05-10 10:49:35][root                ][stage_datasets           ] [local_rank 1/4] Staging ssv2 (multipart_tar)
+[INFO    ][2026-05-10 10:50:44][app.vjepa.train     ][main                     ] Staged datasets: ['/scratch-node/dcanez.22613067/kinetics_240/train.csv', '/scratch-node/dcanez.22613067/ssv2/train.csv']
+[INFO    ][2026-05-10 10:51:11][experiments.stmodels.vision_transformers_v3][vit_large_patch14_capi_lvd1689m] Loading pretrained weights for vit_large_patch14_capi_lvd1689m from https://dl.fbaipublicfiles.com/capi/capi_vitl14_lvd.pth
+[INFO    ][2026-05-10 10:51:23][root                ][_build_frozen_2d_target  ] Loaded frozen_2d target from timm: vit_large_patch14_capi.lvd1689m
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] ViTMultiSeqWrapper(
+  (backbone): UJEPAside(
+    (patch_embed): PatchEmbed3D(
+      (proj): Conv3d(3, 1024, kernel_size=(1, 14, 14), stride=(1, 14, 14))
+    )
+    (rope): CAPI2DRoPE()
+    (blocks): ModuleList(
+      (0-23): 24 x Block(
+        (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (rope_impl): CAPI2DRoPE()
+        (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+        (drop_path1): Identity()
+        (drop_path2): Identity()
+        (attn): Attention(
+          (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+          (attn_drop): Dropout(p=0.0, inplace=False)
+          (proj): Linear(in_features=1024, out_features=1024, bias=False)
+          (proj_drop): Dropout(p=0.0, inplace=False)
+          (rope_impl): CAPI2DRoPE()
+        )
+        (mlp): MLP(
+          (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+          (act): GELU(approximate='none')
+          (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+          (drop): Dropout(p=0.0, inplace=False)
+        )
+      )
+    )
+    (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=False)
+    (st_blocks): ModuleList(
+      (0-23): 24 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (k_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (v_proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=False)
+            (rope_impl): CAPI3DRoPE()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=4096, out_features=1024, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (st_rope): CAPI3DRoPE()
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] UJEPAsidePredictorMultiSeqWrapper(
+  (backbone): PredictorV2(
+    (predictor_embed): Linear(in_features=1024, out_features=384, bias=True)
+    (mask_tokens): ParameterList(
+        (0): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (1): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (2): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+        (3): Parameter containing: [torch.float32 of size 1x384 (cuda:0)]
+    )
+    (predictor_blocks): ModuleList(
+      (0-5): 6 x Block(
+        (residual1): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): Attention(
+            (q_proj): Linear(in_features=384, out_features=384, bias=False)
+            (k_proj): Linear(in_features=384, out_features=384, bias=False)
+            (v_proj): Linear(in_features=384, out_features=384, bias=False)
+            (proj): Linear(in_features=384, out_features=384, bias=False)
+            (rope): Rope()
+          )
+        )
+        (residual2): EfficientResidual(
+          (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+          (fn): MLP(
+            (fc1): Linear(in_features=384, out_features=1536, bias=False)
+            (act): GELU(approximate='none')
+            (fc2): Linear(in_features=1536, out_features=384, bias=False)
+            (drop): Dropout(p=0.0, inplace=False)
+          )
+        )
+      )
+    )
+    (predictor_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+    (predictor_proj): Linear(in_features=384, out_features=1024, bias=True)
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] MultiSeqWrapper(
+  (backbone): Frozen2DTargetWrapper(
+    (backbone): Eva(
+      (patch_embed): PatchEmbed(
+        (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
+        (norm): Identity()
+      )
+      (pos_drop): Dropout(p=0.0, inplace=False)
+      (norm_pre): Identity()
+      (blocks): ModuleList(
+        (0-23): 24 x EvaBlock(
+          (norm1): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (attn): EvaAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path1): Identity()
+          (norm2): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+          (mlp): Mlp(
+            (fc1): Linear(in_features=1024, out_features=4096, bias=True)
+            (act): GELU(approximate='none')
+            (drop1): Dropout(p=0.0, inplace=False)
+            (norm): Identity()
+            (fc2): Linear(in_features=4096, out_features=1024, bias=True)
+            (drop2): Dropout(p=0.0, inplace=False)
+          )
+          (drop_path2): Identity()
+        )
+      )
+      (norm): RMSNorm((1024,), eps=1e-05, elementwise_affine=True)
+      (fc_norm): Identity()
+      (head_drop): Dropout(p=0.0, inplace=False)
+      (head): Identity()
+      (rope): _CapiPatchRoPE()
+    )
+  )
+)
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Encoder number of parameters: 302661632
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Predictor number of parameters: 11416192
+[INFO    ][2026-05-10 10:51:23][root                ][init_video_model         ] Target encoder number of parameters: 0
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset dataset created
+[INFO    ][2026-05-10 10:51:28][WeightedSampler     ][__init__                 ] Using DistributedWeightedSampler with rank 9 / 16
+[INFO    ][2026-05-10 10:51:28][root                ][make_videodataset        ] VideoDataset unsupervised data loader created
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] iterations per epoch/dataset length: 300/399
+[INFO    ][2026-05-10 10:51:28][app.vjepa.train     ][main                     ] Wrapping models in DDP (rank 9)...
+[INFO    ][2026-05-10 10:52:01][app.vjepa.train     ][main                     ] Initializing loader...
+submitit ERROR (2026-05-10 14:06:41,937) - Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:41][submitit            ][process_job              ] Submitted job triggered an exception
+[ERROR   ][2026-05-10 14:06:43][app.vjepa.train     ][_save_crash              ] Saved crash log to /scratch-shared/dcanez/runs/vjepa/ujepaside_capi_lvd/c001_vitl_k32_simple_cross_stable/crash_rank9.log