fix: replace hanging upload_large_folder with per-dir upload + rate limiting

upload_large_folder hangs after ~50 commits (known issue). Reverted to
per-directory upload_folder approach with:
- Split dirs >3000 files into sub-dirs recursively
- 3s delay between uploads (rate limiting)
- Retry with backoff (3 attempts, 5s/10s/15s)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

entrypoint.py

@@ -259,27 +259,110 @@ def save_and_upload():
                     shutil.rmtree(full, ignore_errors=True)
                     dirnames.remove(d)
 
-    # Upload entire /data/ to HF dataset using upload_large_folder.
-    # Handles chunking, dedup, retries, and multi-commit internally.
+    # Upload per-directory with retry + rate limiting
     from huggingface_hub import HfApi
     api = HfApi(token=HF_TOKEN)
-    t0_up = time.time()
-    try:
-        rc, count = run(f"find {PERSIST_PATH} -type f -not -path '*/.cache/*' -not -path '*/.git/*' | wc -l")
-        log(f"  uploading {count.strip()} files via upload_large_folder ...")
-        api.upload_large_folder(
-            folder_path=PERSIST_PATH,
-            repo_id=HF_DATASET_REPO,
-            repo_type="dataset",
-            ignore_patterns=UPLOAD_IGNORE,
-            print_report=False,
-        )
-        elapsed_up = time.time() - t0_up
-        log(f"══ SYNC: done ({elapsed_up:.1f}s) ══")
-    except Exception as e:
-        elapsed_up = time.time() - t0_up
-        err_short = str(e).split('\n')[0][:120]
-        log(f"══ SYNC: upload failed ({elapsed_up:.1f}s): {err_short} ══")
+    ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
+    t0_all = time.time()
+    ok_count = 0
+    fail_count = 0
+
+    def count_files(path):
+        try:
+            return int(subprocess.check_output(
+                f"find '{path}' -type f | wc -l", shell=True, text=True).strip())
+        except Exception:
+            return 0
+
+    def upload_one(local_path, repo_path, max_retries=3):
+        """Upload a single directory with retry + backoff."""
+        nonlocal ok_count, fail_count
+        fc = count_files(local_path)
+        if fc == 0:
+            return
+        for attempt in range(max_retries):
+            t0 = time.time()
+            try:
+                api.upload_folder(
+                    folder_path=local_path,
+                    repo_id=HF_DATASET_REPO,
+                    repo_type="dataset",
+                    path_in_repo=repo_path,
+                    commit_message=f"sync {ts}: {repo_path}/",
+                    ignore_patterns=UPLOAD_IGNORE,
+                )
+                log(f"  {repo_path}/ ok ({time.time()-t0:.1f}s, {fc} files)")
+                ok_count += 1
+                return
+            except Exception as e:
+                err = str(e).split('\n')[0][:100]
+                if attempt < max_retries - 1:
+                    wait = (attempt + 1) * 5
+                    log(f"  {repo_path}/ retry {attempt+1} ({time.time()-t0:.1f}s): {err}")
+                    time.sleep(wait)
+                else:
+                    log(f"  {repo_path}/ FAILED ({time.time()-t0:.1f}s): {err}")
+                    fail_count += 1
+
+    def upload_dir(local_path, repo_path):
+        """Upload directory, splitting into sub-dirs if >3000 files."""
+        fc = count_files(local_path)
+        if fc == 0:
+            return
+        if fc <= 3000:
+            upload_one(local_path, repo_path)
+            time.sleep(3)  # rate limit
+        else:
+            log(f"  {repo_path}/ ({fc} files) → splitting into sub-dirs")
+            # Upload loose files at this level
+            loose = [f for f in os.listdir(local_path)
+                     if os.path.isfile(os.path.join(local_path, f))
+                     and not f.startswith('.')]
+            if loose:
+                for f in loose:
+                    try:
+                        api.upload_file(
+                            path_or_fileobj=os.path.join(local_path, f),
+                            path_in_repo=f"{repo_path}/{f}",
+                            repo_id=HF_DATASET_REPO,
+                            repo_type="dataset",
+                            commit_message=f"sync {ts}: {repo_path}/{f}",
+                        )
+                    except Exception:
+                        pass
+            # Recurse into sub-dirs
+            for d in sorted(os.listdir(local_path)):
+                dp = os.path.join(local_path, d)
+                if os.path.isdir(dp) and not d.startswith('.'):
+                    rp = f"{repo_path}/{d}"
+                    upload_dir(dp, rp)
+
+    # Upload root-level files
+    root_files = [f for f in os.listdir(PERSIST_PATH)
+                  if os.path.isfile(os.path.join(PERSIST_PATH, f))
+                  and not f.startswith('.')]
+    for rf in root_files:
+        try:
+            api.upload_file(
+                path_or_fileobj=os.path.join(PERSIST_PATH, rf),
+                path_in_repo=rf,
+                repo_id=HF_DATASET_REPO,
+                repo_type="dataset",
+                commit_message=f"sync {ts}: {rf}",
+            )
+        except Exception as e:
+            log(f"  {rf} failed: {str(e)[:80]}")
+    if root_files:
+        log(f"  uploaded {len(root_files)} root files")
+
+    # Upload each top-level directory
+    for d in sorted(os.listdir(PERSIST_PATH)):
+        dp = os.path.join(PERSIST_PATH, d)
+        if os.path.isdir(dp) and not d.startswith('.'):
+            upload_dir(dp, d)
+
+    elapsed_all = time.time() - t0_all
+    log(f"══ SYNC: done ({elapsed_all:.1f}s) — {ok_count} ok, {fail_count} failed ══")
 
 
 # Event to signal restore completion (sync must wait)