auto-sync 2026-07-03T16:33:17Z workspace (part 2)

workspace/outputs/hf_sync/hf_push_daemon_login.out

@@ -2988,3 +2988,57 @@ Uploading... 8,448/28,664 files checked, 126/126 uploaded (0.00B transferred), 7
 Uploading... 11,008/28,664 files checked, 126/126 uploaded (0.00B transferred), 10,312 committed in 2 commit(s)
 Uploading... 14,848/28,664 files checked, 126/126 uploaded (0.00B transferred), 14,240 committed in 2 commit(s)
 Uploading... 19,456/28,664 files checked, 126/126 uploaded (0.00B transferred), 18,556 committed in 2 commit(s)
+Uploading... 22,528/28,664 files checked, 136/136 uploaded (0.00B transferred), 21,994 committed in 3 commit(s)
+Uploading... 25,088/28,664 files checked, 136/136 uploaded (0.00B transferred), 24,396 committed in 4 commit(s)
+Uploading... 28,664/28,664 files checked, 927/927 uploaded (0.00B transferred), 27,755 committed in 5 commit(s)
+Upload completed in 7 commits.
+url=https://huggingface.co/anhtld/vla/commit/8ed2ec379ce35f462b7f5b1ab5c84a6370ef46e2
+[2026-07-03T16:32:03Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs -> anhtld/vla/scratch/experiments/dovla_h16_policy_ckpt_runs
+Found 1,276 files to upload
+Uploading... 256/1,276 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/1,276 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/1,276 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/1,276 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 512/1,276 files checked, 46/49 uploaded (0.00B transferred), 250 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 131/138 uploaded (47.5MB transferred), 754 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/a6e1dab15ef3ea799a2f7a1f3940e0d5e2073c2a
+[2026-07-03T16:35:03Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs -> anhtld/vla/scratch/experiments/dovla_h16_rollout_runs
+Found 51 files to upload
+Uploading... 51/51 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 51/51 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 51/51 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/a6e1dab15ef3ea799a2f7a1f3940e0d5e2073c2a
+[2026-07-03T16:36:17Z] upload /scratch/knguy52/dovla/experiments/h16_merged_dataset -> anhtld/vla/scratch/experiments/h16_merged_dataset
+Found 49 files to upload
+Uploading... 49/49 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/a6e1dab15ef3ea799a2f7a1f3940e0d5e2073c2a
+[2026-07-03T16:36:27Z] upload /scratch/knguy52/dovla/experiments/six_task_h16_collection -> anhtld/vla/scratch/experiments/six_task_h16_collection
+Found 87 files to upload
+Uploading... 87/87 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/a6e1dab15ef3ea799a2f7a1f3940e0d5e2073c2a
+[2026-07-03T16:36:40Z] upload /scratch/knguy52/dovla/experiments/embedding_cache -> anhtld/vla/scratch/experiments/embedding_cache
+Found 0 files to upload
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/a6e1dab15ef3ea799a2f7a1f3940e0d5e2073c2a
+[2026-07-03T16:36:40Z] upload /scratch/knguy52/dovla/cache -> anhtld/vla/scratch/cache
+Found 10 files to upload
+Uploading... 10/10 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 10/10 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 10/10 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/a6e1dab15ef3ea799a2f7a1f3940e0d5e2073c2a
+[2026-07-03T16:38:10Z] upload /scratch/knguy52/dovla/caches -> anhtld/vla/scratch/caches
+Found 3 files to upload
+Uploading... 3/3 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/a6e1dab15ef3ea799a2f7a1f3940e0d5e2073c2a
+[2026-07-03T16:38:26Z] upload /lustre09/project/6037638/knguy52/vla/outputs/hf_sync/last_manifest.json -> anhtld/vla/manifests/hf_sync_last_manifest.json
+/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/cli/upload.py:189: UserWarning: Ignoring --exclude since a single file is uploaded.
+  warnings.warn("Ignoring --exclude since a single file is uploaded.")
+url=https://huggingface.co/anhtld/vla/commit/0fdc06e61e972890b40a155063840c6e288bbf0f
+[2026-07-03T16:38:32Z] sync complete
+[2026-07-03T16:38:32Z] sync cycle succeeded

workspace/outputs/hf_sync/hf_push_every_15m.setsid.log

@@ -693,3 +693,12 @@ url=https://huggingface.co/anhtld/vla/commit/4d112859bca406ec3d7b3098bb99ced87bf
 [2026-07-03T16:16:20Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla -> anhtld/vla/workspace
 Terminated
 [2026-07-03T16:18:16Z] sync cycle failed with status=143
+[2026-07-03T16:33:16Z] checking Hugging Face auth
+[2026-07-03T16:33:16Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-03T16:33:17Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla -> anhtld/vla/workspace
+Found 28,796 files to upload
+Uploading... 0/28,796 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/28,796 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 1,024/28,796 files checked, 14/14 uploaded (0.00B transferred), 250 committed in 0 commit(s)
+Uploading... 3,072/28,796 files checked, 64/64 uploaded (0.00B transferred), 2,466 committed in 1 commit(s)
+Uploading... 5,888/28,796 files checked, 121/121 uploaded (0.00B transferred), 5,236 committed in 1 commit(s)

workspace/outputs/hf_sync/hf_sync.log

@@ -788,3 +788,15 @@
 [2026-07-03T16:16:16Z] checking Hugging Face auth
 [2026-07-03T16:16:20Z] sync start repo=anhtld/vla type=model dry_run=0
 [2026-07-03T16:16:20Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla -> anhtld/vla/workspace
+[2026-07-03T16:32:03Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs -> anhtld/vla/scratch/experiments/dovla_h16_policy_ckpt_runs
+[2026-07-03T16:33:16Z] checking Hugging Face auth
+[2026-07-03T16:33:16Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-03T16:33:17Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla -> anhtld/vla/workspace
+[2026-07-03T16:35:03Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs -> anhtld/vla/scratch/experiments/dovla_h16_rollout_runs
+[2026-07-03T16:36:17Z] upload /scratch/knguy52/dovla/experiments/h16_merged_dataset -> anhtld/vla/scratch/experiments/h16_merged_dataset
+[2026-07-03T16:36:27Z] upload /scratch/knguy52/dovla/experiments/six_task_h16_collection -> anhtld/vla/scratch/experiments/six_task_h16_collection
+[2026-07-03T16:36:40Z] upload /scratch/knguy52/dovla/experiments/embedding_cache -> anhtld/vla/scratch/experiments/embedding_cache
+[2026-07-03T16:36:40Z] upload /scratch/knguy52/dovla/cache -> anhtld/vla/scratch/cache
+[2026-07-03T16:38:10Z] upload /scratch/knguy52/dovla/caches -> anhtld/vla/scratch/caches
+[2026-07-03T16:38:26Z] upload /lustre09/project/6037638/knguy52/vla/outputs/hf_sync/last_manifest.json -> anhtld/vla/manifests/hf_sync_last_manifest.json
+[2026-07-03T16:38:32Z] sync complete

workspace/outputs/hf_sync/hf_sync_daemon.log

@@ -75,3 +75,4 @@
 [2026-07-03T15:51:50Z] sync cycle succeeded
 [2026-07-03T16:01:16Z] sync cycle succeeded
 [2026-07-03T16:18:16Z] sync cycle failed with status=143
+[2026-07-03T16:38:32Z] sync cycle succeeded

workspace/outputs/hf_sync/last_manifest.json

@@ -1,5 +1,5 @@
 {
-  "generated_utc": "2026-07-03T16:16:20.688591+00:00",
+  "generated_utc": "2026-07-03T16:33:17.067502+00:00",
   "repo_id": "anhtld/vla",
   "project_dir": "/home/knguy52/links/projects/def-yalda/knguy52/vla",
   "scratch_root": "/scratch/knguy52/dovla",