auto-sync 2026-07-03T16:06:56Z workspace (part 3)

workspace/outputs/hf_sync/hf_push_daemon_login.out

@@ -2969,3 +2969,22 @@ url=https://huggingface.co/anhtld/vla/commit/3890d058fd07e8fd278e936657cc874e3ef
 url=https://huggingface.co/anhtld/vla/commit/df983f14be27dc82152e7bea96554dbb1eb2a2d7
 [2026-07-03T15:51:50Z] sync complete
 [2026-07-03T15:51:50Z] sync cycle succeeded
+[2026-07-03T16:06:50Z] checking Hugging Face auth
+[2026-07-03T16:06:55Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-03T16:06:56Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace
+Found 28,664 files to upload
+Uploading... 256/28,664 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/28,664 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/28,664 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/28,664 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/28,664 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/28,664 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/28,664 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 256/28,664 files checked, 0/0 uploaded (0.00B transferred), 0 committed in 0 commit(s)
+Uploading... 768/28,664 files checked, 14/14 uploaded (0.00B transferred), 180 committed in 0 commit(s)
+Uploading... 3,072/28,664 files checked, 64/64 uploaded (0.00B transferred), 2,268 committed in 2 commit(s)
+Uploading... 5,632/28,664 files checked, 121/121 uploaded (0.00B transferred), 4,932 committed in 2 commit(s)
+Uploading... 8,448/28,664 files checked, 126/126 uploaded (0.00B transferred), 7,714 committed in 2 commit(s)
+Uploading... 11,008/28,664 files checked, 126/126 uploaded (0.00B transferred), 10,312 committed in 2 commit(s)
+Uploading... 14,848/28,664 files checked, 126/126 uploaded (0.00B transferred), 14,240 committed in 2 commit(s)
+Uploading... 19,456/28,664 files checked, 126/126 uploaded (0.00B transferred), 18,556 committed in 2 commit(s)

workspace/outputs/hf_sync/hf_push_every_15m.setsid.log

@@ -688,3 +688,8 @@ No files have been modified since last commit. Skipping to prevent empty commit.
 url=https://huggingface.co/anhtld/vla/commit/4d112859bca406ec3d7b3098bb99ced87bfd0cd7
 [2026-07-03T16:01:16Z] sync complete
 [2026-07-03T16:01:16Z] sync cycle succeeded
+[2026-07-03T16:16:16Z] checking Hugging Face auth
+[2026-07-03T16:16:20Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-03T16:16:20Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla -> anhtld/vla/workspace
+Terminated
+[2026-07-03T16:18:16Z] sync cycle failed with status=143

workspace/outputs/hf_sync/hf_sync.log

@@ -782,3 +782,9 @@
 [2026-07-03T16:01:09Z] upload /scratch/knguy52/dovla/caches -> anhtld/vla/scratch/caches
 [2026-07-03T16:01:13Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla/outputs/hf_sync/last_manifest.json -> anhtld/vla/manifests/hf_sync_last_manifest.json
 [2026-07-03T16:01:16Z] sync complete
+[2026-07-03T16:06:50Z] checking Hugging Face auth
+[2026-07-03T16:06:55Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-03T16:06:56Z] upload /lustre09/project/6037638/knguy52/vla -> anhtld/vla/workspace
+[2026-07-03T16:16:16Z] checking Hugging Face auth
+[2026-07-03T16:16:20Z] sync start repo=anhtld/vla type=model dry_run=0
+[2026-07-03T16:16:20Z] upload /home/knguy52/links/projects/def-yalda/knguy52/vla -> anhtld/vla/workspace

workspace/outputs/hf_sync/hf_sync_daemon.log

@@ -74,3 +74,4 @@
 [2026-07-03T15:27:26Z] sync cycle succeeded
 [2026-07-03T15:51:50Z] sync cycle succeeded
 [2026-07-03T16:01:16Z] sync cycle succeeded
+[2026-07-03T16:18:16Z] sync cycle failed with status=143

workspace/outputs/hf_sync/last_manifest.json

@@ -1,5 +1,5 @@
 {
-  "generated_utc": "2026-07-03T15:42:31.609619+00:00",
+  "generated_utc": "2026-07-03T16:16:20.688591+00:00",
   "repo_id": "anhtld/vla",
   "project_dir": "/home/knguy52/links/projects/def-yalda/knguy52/vla",
   "scratch_root": "/scratch/knguy52/dovla",