fix: auto-split large dirs + exclude huggingrun.env from upload

- Dirs with >2000 files are recursively split into sub-dirs
(avoids 504 timeout on usr/ which has 10K+ files)
- Exclude huggingrun.env (contains HF_TOKEN, rejected by secret scanner)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

entrypoint.py

@@ -51,6 +51,7 @@ UPLOAD_IGNORE = [
     "*.sock", "*.lock",
     ".huggingface",
     ".cache",
+    "huggingrun.env",   # contains HF_TOKEN, rejected by secret scanner
 ]
 
 
@@ -249,55 +250,96 @@ def save_and_upload():
                     shutil.rmtree(full, ignore_errors=True)
                     dirnames.remove(d)
 
-    # Upload per top-level directory (avoids 504 timeout on huge single commit)
+    # Upload directory tree — split large dirs into sub-dirs to avoid 504 timeout
     api = HfApi(token=HF_TOKEN)
     ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
-
-    # First, upload loose files in /data/ root (user-packages.list, etc.)
     t0_all = time.time()
-    try:
-        root_files = [f for f in os.listdir(PERSIST_PATH)
-                      if os.path.isfile(os.path.join(PERSIST_PATH, f))
-                      and not f.startswith('.')]
-        if root_files:
-            for rf in root_files:
-                fpath = os.path.join(PERSIST_PATH, rf)
-                api.upload_file(
-                    path_or_fileobj=fpath,
-                    path_in_repo=rf,
+    uploaded = 0
+    failed = 0
+
+    def upload_dir(local_path, repo_path):
+        """Upload a directory. If it's too large (>2000 files), split into sub-dirs."""
+        nonlocal uploaded, failed
+        # Count files (excluding ignored patterns)
+        try:
+            file_count = int(subprocess.check_output(
+                f"find '{local_path}' -type f -not -name '*.pyc' -not -path '*/__pycache__/*' "
+                f"-not -path '*/.cache/*' -not -path '*/.git/*' | wc -l",
+                shell=True, text=True).strip())
+        except Exception:
+            file_count = 0
+
+        if file_count > 2000:
+            # Too large — recurse into sub-directories
+            log(f"  {repo_path}/ ({file_count} files) → splitting")
+            sub_items = sorted(os.listdir(local_path))
+            # Upload files in this dir first (non-recursive)
+            dir_files = [f for f in sub_items
+                         if os.path.isfile(os.path.join(local_path, f))
+                         and not f.startswith('.')]
+            if dir_files:
+                for f in dir_files:
+                    try:
+                        api.upload_file(
+                            path_or_fileobj=os.path.join(local_path, f),
+                            path_in_repo=f"{repo_path}/{f}" if repo_path else f,
+                            repo_id=HF_DATASET_REPO,
+                            repo_type="dataset",
+                            commit_message=f"sync {ts}: {repo_path}/{f}",
+                        )
+                    except Exception:
+                        pass
+            # Recurse into sub-dirs
+            for d in sub_items:
+                dp = os.path.join(local_path, d)
+                if os.path.isdir(dp) and not d.startswith('.'):
+                    rp = f"{repo_path}/{d}" if repo_path else d
+                    upload_dir(dp, rp)
+        else:
+            # Small enough — upload as single commit
+            t0 = time.time()
+            try:
+                api.upload_folder(
+                    folder_path=local_path,
                     repo_id=HF_DATASET_REPO,
                     repo_type="dataset",
-                    commit_message=f"sync {ts}: {rf}",
+                    path_in_repo=repo_path,
+                    commit_message=f"sync {ts}: {repo_path}/",
+                    ignore_patterns=UPLOAD_IGNORE,
                 )
-            log(f"  uploaded {len(root_files)} root files")
-    except Exception as e:
-        log(f"  root files upload failed: {e}")
-
-    # Then upload each top-level directory as a separate commit
-    top_dirs = sorted([d for d in os.listdir(PERSIST_PATH)
-                       if os.path.isdir(os.path.join(PERSIST_PATH, d))
-                       and not d.startswith('.')])
-    uploaded = 0
-    failed = 0
-    for d in top_dirs:
-        dir_path = os.path.join(PERSIST_PATH, d)
-        t0 = time.time()
+                elapsed = time.time() - t0
+                log(f"  {repo_path}/ uploaded ({elapsed:.1f}s, {file_count} files)")
+                uploaded += 1
+            except Exception as e:
+                elapsed = time.time() - t0
+                err_short = str(e).split('\n')[0][:100]
+                log(f"  {repo_path}/ failed ({elapsed:.1f}s): {err_short}")
+                failed += 1
+
+    # Upload loose files at root
+    root_files = [f for f in os.listdir(PERSIST_PATH)
+                  if os.path.isfile(os.path.join(PERSIST_PATH, f))
+                  and not f.startswith('.')]
+    for rf in root_files:
         try:
-            api.upload_folder(
-                folder_path=dir_path,
+            api.upload_file(
+                path_or_fileobj=os.path.join(PERSIST_PATH, rf),
+                path_in_repo=rf,
                 repo_id=HF_DATASET_REPO,
                 repo_type="dataset",
-                path_in_repo=d,
-                commit_message=f"sync {ts}: {d}/",
-                ignore_patterns=UPLOAD_IGNORE,
+                commit_message=f"sync {ts}: {rf}",
             )
-            elapsed = time.time() - t0
-            log(f"  {d}/ uploaded ({elapsed:.1f}s)")
-            uploaded += 1
         except Exception as e:
-            elapsed = time.time() - t0
-            log(f"  {d}/ failed ({elapsed:.1f}s): {e}")
-            failed += 1
+            log(f"  {rf} failed: {str(e)[:80]}")
+    if root_files:
+        log(f"  uploaded {len(root_files)} root files")
+
+    # Upload each top-level directory (auto-splits if >2000 files)
+    top_dirs = sorted([d for d in os.listdir(PERSIST_PATH)
+                       if os.path.isdir(os.path.join(PERSIST_PATH, d))
+                       and not d.startswith('.')])
+    for d in top_dirs:
+        upload_dir(os.path.join(PERSIST_PATH, d), d)
 
     elapsed_all = time.time() - t0_all
     log(f"══ SYNC: done ({elapsed_all:.1f}s) — {uploaded} ok, {failed} failed ══")