auto-sync 2026-07-03T00:30:47Z workspace (part 2)

workspace/outputs/hf_sync/hf_push_daemon_login.out

@@ -995,3 +995,74 @@ Uploading... 1,536/27,444 files checked, 15/15 uploaded (30.5MB transferred), 75
 Uploading... 5,376/27,444 files checked, 113/113 uploaded (30.5MB transferred), 4,720 committed in 1 commit(s)
 Uploading... 10,752/27,444 files checked, 118/118 uploaded (30.5MB transferred), 10,118 committed in 1 commit(s)
 Uploading... 20,736/27,444 files checked, 118/118 uploaded (30.5MB transferred), 19,972 committed in 1 commit(s)
+Uploading... 27,444/27,444 files checked, 901/901 uploaded (30.5MB transferred), 27,097 committed in 2 commit(s)
+Upload completed in 3 commits.
+url=https://huggingface.co/anhtld/vla/commit/eeb1e6f843d7b11bdfee0f9f0d013e416a45552e
+[2026-07-02T23:37:11Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs -> anhtld/vla/scratch/experiments/dovla_h16_policy_ckpt_runs
+Found 1,276 files to upload
+Uploading... 256/1,276 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/1,276 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 512/1,276 files checked, 34/42 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 768/1,276 files checked, 96/101 uploaded (0.00B transferred), 250 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 118/145 uploaded (26.3MB transferred), 524 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 197/202 uploaded (26.3MB transferred), 524 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 197/202 uploaded (26.3MB transferred), 524 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 197/202 uploaded (26.3MB transferred), 524 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 198/202 uploaded (26.3MB transferred), 524 committed in 0 commit(s)
+Uploading... 1,024/1,276 files checked, 198/202 uploaded (26.3MB transferred), 524 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/eeb1e6f843d7b11bdfee0f9f0d013e416a45552e
+[2026-07-02T23:42:26Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs -> anhtld/vla/scratch/experiments/dovla_h16_rollout_runs
+Found 51 files to upload
+Uploading... 51/51 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 51/51 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 51/51 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/eeb1e6f843d7b11bdfee0f9f0d013e416a45552e
+[2026-07-02T23:43:32Z] upload /scratch/knguy52/dovla/experiments/h16_merged_dataset -> anhtld/vla/scratch/experiments/h16_merged_dataset
+Found 49 files to upload
+Uploading... 49/49 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/eeb1e6f843d7b11bdfee0f9f0d013e416a45552e
+[2026-07-02T23:43:45Z] upload /scratch/knguy52/dovla/experiments/six_task_h16_collection -> anhtld/vla/scratch/experiments/six_task_h16_collection
+Found 77 files to upload
+Uploading... 77/77 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/eeb1e6f843d7b11bdfee0f9f0d013e416a45552e
+[2026-07-02T23:43:52Z] upload /scratch/knguy52/dovla/experiments/embedding_cache -> anhtld/vla/scratch/experiments/embedding_cache
+Found 0 files to upload
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/eeb1e6f843d7b11bdfee0f9f0d013e416a45552e
+[2026-07-02T23:43:52Z] upload /scratch/knguy52/dovla/cache -> anhtld/vla/scratch/cache
+Found 10 files to upload
+Uploading... 10/10 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/eeb1e6f843d7b11bdfee0f9f0d013e416a45552e
+[2026-07-02T23:44:13Z] upload /scratch/knguy52/dovla/caches -> anhtld/vla/scratch/caches
+Found 3 files to upload
+Uploading... 3/3 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+No files have been modified since last commit. Skipping to prevent empty commit.
+url=https://huggingface.co/anhtld/vla/commit/eeb1e6f843d7b11bdfee0f9f0d013e416a45552e
+[2026-07-02T23:44:42Z] upload /lustre09/project/6037638/knguy52/vla/outputs/hf_sync/last_manifest.json -> anhtld/vla/manifests/hf_sync_last_manifest.json
+/lustre09/project/6037638/knguy52/vla/.venv/lib/python3.11/site-packages/huggingface_hub/cli/upload.py:189: UserWarning: Ignoring --exclude since a single file is uploaded.
+  warnings.warn("Ignoring --exclude since a single file is uploaded.")
+url=https://huggingface.co/anhtld/vla/commit/4066afdff4f7014d3d33cd9f339948a53c90073c
+[2026-07-02T23:44:47Z] sync complete
+[2026-07-02T23:44:47Z] sync cycle succeeded
+[2026-07-02T23:59:47Z] checking Hugging Face auth
+[2026-07-02T23:59:50Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-02T23:59:50Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace
+Terminated
+[2026-07-03T00:15:44Z] sync cycle failed with status=143
+[2026-07-03T00:30:44Z] checking Hugging Face auth
+[2026-07-03T00:30:46Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-03T00:30:47Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace
+Found 27,556 files to upload
+Uploading... 256/27,556 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/27,556 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 512/27,556 files checked, 2/4 uploaded (565kB transferred), 0 committed in 0 commit(s)
+Uploading... 3,328/27,556 files checked, 58/58 uploaded (920kB transferred), 2,663 committed in 1 commit(s)
+Uploading... 6,912/27,556 files checked, 115/115 uploaded (920kB transferred), 6,249 committed in 1 commit(s)
+Uploading... 10,752/27,556 files checked, 120/120 uploaded (920kB transferred), 9,898 committed in 1 commit(s)
+Uploading... 15,616/27,556 files checked, 120/120 uploaded (920kB transferred), 15,007 committed in 1 commit(s)
+Uploading... 21,504/27,556 files checked, 126/126 uploaded (920kB transferred), 20,703 committed in 1 commit(s)

workspace/outputs/hf_sync/hf_sync.log

@@ -200,3 +200,18 @@
 [2026-07-02T23:06:58Z] checking Hugging Face auth
 [2026-07-02T23:07:00Z] sync start repo=anhtld/vla type=model dry_run=0
 [2026-07-02T23:07:00Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace
+[2026-07-02T23:37:11Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_policy_ckpt_runs -> anhtld/vla/scratch/experiments/dovla_h16_policy_ckpt_runs
+[2026-07-02T23:42:26Z] upload /scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs -> anhtld/vla/scratch/experiments/dovla_h16_rollout_runs
+[2026-07-02T23:43:32Z] upload /scratch/knguy52/dovla/experiments/h16_merged_dataset -> anhtld/vla/scratch/experiments/h16_merged_dataset
+[2026-07-02T23:43:45Z] upload /scratch/knguy52/dovla/experiments/six_task_h16_collection -> anhtld/vla/scratch/experiments/six_task_h16_collection
+[2026-07-02T23:43:52Z] upload /scratch/knguy52/dovla/experiments/embedding_cache -> anhtld/vla/scratch/experiments/embedding_cache
+[2026-07-02T23:43:52Z] upload /scratch/knguy52/dovla/cache -> anhtld/vla/scratch/cache
+[2026-07-02T23:44:13Z] upload /scratch/knguy52/dovla/caches -> anhtld/vla/scratch/caches
+[2026-07-02T23:44:42Z] upload /lustre09/project/6037638/knguy52/vla/outputs/hf_sync/last_manifest.json -> anhtld/vla/manifests/hf_sync_last_manifest.json
+[2026-07-02T23:44:47Z] sync complete
+[2026-07-02T23:59:47Z] checking Hugging Face auth
+[2026-07-02T23:59:50Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-02T23:59:50Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace
+[2026-07-03T00:30:44Z] checking Hugging Face auth
+[2026-07-03T00:30:46Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-03T00:30:47Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace

workspace/outputs/hf_sync/hf_sync_daemon.log

@@ -20,3 +20,5 @@
 [2026-07-02T20:48:13Z] sync cycle succeeded
 [2026-07-02T21:57:13Z] sync cycle succeeded
 [2026-07-02T22:51:58Z] sync cycle succeeded
+[2026-07-02T23:44:47Z] sync cycle succeeded
+[2026-07-03T00:15:44Z] sync cycle failed with status=143

workspace/outputs/hf_sync/last_manifest.json

@@ -1,5 +1,5 @@
 {
-  "generated_utc": "2026-07-02T23:07:00.682476+00:00",
+  "generated_utc": "2026-07-03T00:30:47.499564+00:00",
   "repo_id": "anhtld/vla",
   "project_dir": "/lustre09/project/6037638/knguy52/vla",
   "scratch_root": "/scratch/knguy52/dovla",