auto-sync 2026-07-04T03:59:09Z workspace (part 2)

workspace/outputs/hf_sync/hf_push_daemon_login.out

@@ -4347,3 +4347,70 @@ Uploading... 7,680/31,094 files checked, 129/129 uploaded (0.00B transferred), 6
 Uploading... 10,752/31,094 files checked, 129/129 uploaded (0.00B transferred), 10,032 committed in 1 commit(s)
 Uploading... 14,592/31,094 files checked, 129/129 uploaded (0.00B transferred), 13,979 committed in 1 commit(s)
 Uploading... 19,456/31,094 files checked, 129/129 uploaded (0.00B transferred), 18,639 committed in 1 commit(s)
+Uploading... 22,784/31,094 files checked, 139/139 uploaded (0.00B transferred), 22,081 committed in 2 commit(s)
+Upload completed in 2 commits.
+url=https://huggingface.co/anhtld/vla/commit/aa1fed0020e91fe734157c7d60f6dfc95ad81693
+[2026-07-04T03:36:42Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs -> anhtld/vla/scratch/experiments/dovla_h16_policy_ckpt_runs
+Found 1,276 files to upload
+Uploading... 256/1,276 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 768/1,276 files checked, 62/101 uploaded (0.00B transferred), 250 committed in 0 commit(s)
+Uploading... 768/1,276 files checked, 94/101 uploaded (0.00B transferred), 250 committed in 0 commit(s)
+Uploading... 768/1,276 files checked, 94/101 uploaded (0.00B transferred), 250 committed in 0 commit(s)
+Uploading... 768/1,276 files checked, 99/101 uploaded (26.3MB transferred), 250 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/1260fad0b24f709ba31916ab901c8d3eb46edb89
+[2026-07-04T03:39:42Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs -> anhtld/vla/scratch/experiments/dovla_h16_rollout_runs
+Found 51 files to upload
+Uploading... 51/51 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/1260fad0b24f709ba31916ab901c8d3eb46edb89
+[2026-07-04T03:39:49Z] upload /scratch/knguy52/dovla/experiments/h16_merged_dataset -> anhtld/vla/scratch/experiments/h16_merged_dataset
+Found 49 files to upload
+Uploading... 49/49 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/1260fad0b24f709ba31916ab901c8d3eb46edb89
+[2026-07-04T03:39:56Z] upload /scratch/knguy52/dovla/experiments/six_task_h16_collection -> anhtld/vla/scratch/experiments/six_task_h16_collection
+Found 87 files to upload
+Uploading... 87/87 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/5e8a26cf030526bbc2f7747b8b25c510039a22dd
+[2026-07-04T03:40:09Z] upload /scratch/knguy52/dovla/experiments/embedding_cache -> anhtld/vla/scratch/experiments/embedding_cache
+Found 0 files to upload
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/5e8a26cf030526bbc2f7747b8b25c510039a22dd
+[2026-07-04T03:40:10Z] upload /scratch/knguy52/dovla/cache -> anhtld/vla/scratch/cache
+Found 10 files to upload
+Uploading... 10/10 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 10/10 files checked, 2/4 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 10/10 files checked, 3/4 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/5e8a26cf030526bbc2f7747b8b25c510039a22dd
+[2026-07-04T03:41:21Z] upload /scratch/knguy52/dovla/caches -> anhtld/vla/scratch/caches
+Found 3 files to upload
+Uploading... 3/3 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/5e8a26cf030526bbc2f7747b8b25c510039a22dd
+[2026-07-04T03:41:32Z] upload /lustre09/project/6037638/knguy52/vla/outputs/hf_sync/last_manifest.json -> anhtld/vla/manifests/hf_sync_last_manifest.json
+/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/cli/upload.py:189: UserWarning: Ignoring --exclude since a single file is uploaded.
+  warnings.warn("Ignoring --exclude since a single file is uploaded.")
+url=https://huggingface.co/anhtld/vla/commit/c65161763de5298a746e249e5bdccaf2f68ec84d
+[2026-07-04T03:41:39Z] sync complete
+[2026-07-04T03:41:39Z] sync cycle succeeded
+[2026-07-04T03:56:39Z] checking Hugging Face auth
+[2026-07-04T03:56:42Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-04T03:56:42Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace
+/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/hf_api.py:11299: UserWarning: Warnings while validating metadata in README.md:
+- empty or missing yaml metadata in repo card
+  warnings.warn(f"Warnings while validating metadata in README.md:\n{message}")
+Found 31,188 files to upload
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 5,120/31,188 files checked, 87/87 uploaded (0.00B transferred), 4,585 committed in 1 commit(s)
+Uploading... 13,312/31,188 files checked, 129/129 uploaded (0.00B transferred), 12,711 committed in 1 commit(s)

workspace/outputs/hf_sync/hf_push_every_15m.setsid.log

@@ -2026,3 +2026,76 @@ Uploading... 7,680/31,094 files checked, 129/129 uploaded (240kB transferred), 6
 Uploading... 10,752/31,094 files checked, 129/129 uploaded (240kB transferred), 10,030 committed in 1 commit(s)
 Uploading... 14,592/31,094 files checked, 129/129 uploaded (240kB transferred), 13,978 committed in 1 commit(s)
 Uploading... 19,456/31,094 files checked, 129/129 uploaded (240kB transferred), 18,636 committed in 1 commit(s)
+Uploading... 22,784/31,094 files checked, 139/139 uploaded (240kB transferred), 22,079 committed in 2 commit(s)
+Upload completed in 2 commits.
+url=https://huggingface.co/anhtld/vla/commit/564ab00e4a092b79487e5ac3aca5b750cd7f8c35
+[2026-07-04T03:36:50Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs -> anhtld/vla/scratch/experiments/dovla_h16_policy_ckpt_runs
+Found 1,276 files to upload
+Uploading... 256/1,276 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/1,276 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 104/124 uploaded (28.2MB transferred), 524 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 104/124 uploaded (28.2MB transferred), 524 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 132/202 uploaded (47.5MB transferred), 754 committed in 0 commit(s)
+Uploading... 1,276/1,276 files checked, 159/233 uploaded (47.5MB transferred), 754 committed in 0 commit(s)
+Uploading... 1,276/1,276 files checked, 243/254 uploaded (47.5MB transferred), 754 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/5e8a26cf030526bbc2f7747b8b25c510039a22dd
+[2026-07-04T03:40:32Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs -> anhtld/vla/scratch/experiments/dovla_h16_rollout_runs
+Found 51 files to upload
+Uploading... 51/51 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/5e8a26cf030526bbc2f7747b8b25c510039a22dd
+[2026-07-04T03:40:59Z] upload /scratch/knguy52/dovla/experiments/h16_merged_dataset -> anhtld/vla/scratch/experiments/h16_merged_dataset
+Found 49 files to upload
+Uploading... 49/49 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/5e8a26cf030526bbc2f7747b8b25c510039a22dd
+[2026-07-04T03:41:06Z] upload /scratch/knguy52/dovla/experiments/six_task_h16_collection -> anhtld/vla/scratch/experiments/six_task_h16_collection
+Found 87 files to upload
+Uploading... 87/87 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/5e8a26cf030526bbc2f7747b8b25c510039a22dd
+[2026-07-04T03:41:36Z] upload /scratch/knguy52/dovla/experiments/embedding_cache -> anhtld/vla/scratch/experiments/embedding_cache
+Found 0 files to upload
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/5e8a26cf030526bbc2f7747b8b25c510039a22dd
+[2026-07-04T03:41:36Z] upload /scratch/knguy52/dovla/cache -> anhtld/vla/scratch/cache
+Found 10 files to upload
+Uploading... 10/10 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 10/10 files checked, 2/4 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 10/10 files checked, 3/4 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 10/10 files checked, 3/4 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/c65161763de5298a746e249e5bdccaf2f68ec84d
+[2026-07-04T03:43:09Z] upload /scratch/knguy52/dovla/caches -> anhtld/vla/scratch/caches
+Found 3 files to upload
+Uploading... 3/3 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 3/3 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/8e0a5ca190f9e7631cedb7d0c8c7bf19a637b006
+[2026-07-04T03:44:07Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla/outputs/hf_sync/last_manifest.json -> anhtld/vla/manifests/hf_sync_last_manifest.json
+/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/cli/upload.py:189: UserWarning: Ignoring --exclude since a single file is uploaded.
+  warnings.warn("Ignoring --exclude since a single file is uploaded.")
+Removing 1 file(s) from commit that have not changed.
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/e760541c3aafba039ad2e67ba802ea06d5f2c102
+[2026-07-04T03:44:08Z] sync complete
+[2026-07-04T03:44:08Z] sync cycle succeeded
+[2026-07-04T03:59:08Z] checking Hugging Face auth
+[2026-07-04T03:59:08Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-04T03:59:09Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla -> anhtld/vla/workspace
+/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/hf_api.py:11299: UserWarning: Warnings while validating metadata in README.md:
+- empty or missing yaml metadata in repo card
+  warnings.warn(f"Warnings while validating metadata in README.md:\n{message}")
+Found 31,188 files to upload
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/31,188 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 1,024/31,188 files checked, 25/25 uploaded (221kB transferred), 490 committed in 1 commit(s)
+Uploading... 3,840/31,188 files checked, 87/87 uploaded (221kB transferred), 3,061 committed in 1 commit(s)
+Uploading... 6,912/31,188 files checked, 124/124 uploaded (221kB transferred), 6,200 committed in 1 commit(s)
+Uploading... 10,240/31,188 files checked, 129/129 uploaded (221kB transferred), 9,405 committed in 1 commit(s)
+Uploading... 22,016/31,188 files checked, 139/139 uploaded (221kB transferred), 21,478 committed in 1 commit(s)

workspace/outputs/hf_sync/hf_sync.log

@@ -1272,3 +1272,27 @@
 [2026-07-04T03:26:40Z] checking Hugging Face auth
 [2026-07-04T03:26:42Z] sync start repo=anhtld/vla type=model dry_run=0
 [2026-07-04T03:26:42Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace
+[2026-07-04T03:36:42Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs -> anhtld/vla/scratch/experiments/dovla_h16_policy_ckpt_runs
+[2026-07-04T03:36:50Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs -> anhtld/vla/scratch/experiments/dovla_h16_policy_ckpt_runs
+[2026-07-04T03:39:42Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs -> anhtld/vla/scratch/experiments/dovla_h16_rollout_runs
+[2026-07-04T03:39:49Z] upload /scratch/knguy52/dovla/experiments/h16_merged_dataset -> anhtld/vla/scratch/experiments/h16_merged_dataset
+[2026-07-04T03:39:56Z] upload /scratch/knguy52/dovla/experiments/six_task_h16_collection -> anhtld/vla/scratch/experiments/six_task_h16_collection
+[2026-07-04T03:40:09Z] upload /scratch/knguy52/dovla/experiments/embedding_cache -> anhtld/vla/scratch/experiments/embedding_cache
+[2026-07-04T03:40:10Z] upload /scratch/knguy52/dovla/cache -> anhtld/vla/scratch/cache
+[2026-07-04T03:40:32Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs -> anhtld/vla/scratch/experiments/dovla_h16_rollout_runs
+[2026-07-04T03:40:59Z] upload /scratch/knguy52/dovla/experiments/h16_merged_dataset -> anhtld/vla/scratch/experiments/h16_merged_dataset
+[2026-07-04T03:41:06Z] upload /scratch/knguy52/dovla/experiments/six_task_h16_collection -> anhtld/vla/scratch/experiments/six_task_h16_collection
+[2026-07-04T03:41:21Z] upload /scratch/knguy52/dovla/caches -> anhtld/vla/scratch/caches
+[2026-07-04T03:41:32Z] upload /lustre09/project/6037638/knguy52/vla/outputs/hf_sync/last_manifest.json -> anhtld/vla/manifests/hf_sync_last_manifest.json
+[2026-07-04T03:41:36Z] upload /scratch/knguy52/dovla/experiments/embedding_cache -> anhtld/vla/scratch/experiments/embedding_cache
+[2026-07-04T03:41:36Z] upload /scratch/knguy52/dovla/cache -> anhtld/vla/scratch/cache
+[2026-07-04T03:41:39Z] sync complete
+[2026-07-04T03:43:09Z] upload /scratch/knguy52/dovla/caches -> anhtld/vla/scratch/caches
+[2026-07-04T03:44:07Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla/outputs/hf_sync/last_manifest.json -> anhtld/vla/manifests/hf_sync_last_manifest.json
+[2026-07-04T03:44:08Z] sync complete
+[2026-07-04T03:56:39Z] checking Hugging Face auth
+[2026-07-04T03:56:42Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-04T03:56:42Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace
+[2026-07-04T03:59:08Z] checking Hugging Face auth
+[2026-07-04T03:59:08Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-04T03:59:09Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla -> anhtld/vla/workspace

workspace/outputs/hf_sync/hf_sync_daemon.log

@@ -117,3 +117,5 @@
 [2026-07-04T02:53:02Z] sync cycle failed with status=1
 [2026-07-04T03:11:20Z] sync cycle succeeded
 [2026-07-04T03:11:40Z] sync cycle succeeded
+[2026-07-04T03:41:39Z] sync cycle succeeded
+[2026-07-04T03:44:08Z] sync cycle succeeded

workspace/outputs/hf_sync/last_manifest.json

@@ -1,12 +1,12 @@
 {
-  "generated_utc": "2026-07-04T03:26:42.828520+00:00",
+  "generated_utc": "2026-07-04T03:59:09.438692+00:00",
   "repo_id": "anhtld/vla",
-  "project_dir": "/lustre09/project/6037638/knguy52/vla",
+  "project_dir": "/home/knguy52/links/projects/def-yalda/knguy52/vla",
   "scratch_root": "/scratch/knguy52/dovla",
   "roots": [
     {
       "remote": "workspace",
-      "local": "/lustre09/project/6037638/knguy52/vla",
+      "local": "/home/knguy52/links/projects/def-yalda/knguy52/vla",
       "exists": true
     },
     {