revert: restore original upload script

scripts/upload_embeddings.py

@@ -1,9 +1,7 @@
 """Upload embeddings to HuggingFace Space"""
 import sys
-import os
-import time
 from pathlib import Path
-from huggingface_hub import HfApi, CommitOperationAdd
+from huggingface_hub import HfApi
 from dotenv import load_dotenv
 
 # Add parent directory to path
@@ -33,57 +31,6 @@ def verify_space_access():
         return False
 
 
-def wait_for_file_upload(api: HfApi, file_path: Path, repo_path: str):
-    """Wait for a file to appear in the repository"""
-    max_attempts = 30  # 5 minutes total
-    for attempt in range(max_attempts):
-        try:
-            # Check if file exists in repo
-            files = api.list_repo_files(
-                repo_id=Settings.HF_SPACE,
-                repo_type="space"
-            )
-            if repo_path in files:
-                return True
-            
-            print(f"Waiting for {file_path.name} to appear in repository... ({attempt + 1}/{max_attempts})")
-            time.sleep(10)  # Wait 10 seconds between checks
-            
-        except Exception as e:
-            print(f"Error checking file status: {str(e)}")
-    
-    return False
-
-
-def upload_large_file(api: HfApi, file_path: Path, repo_path: str):
-    """Upload a single large file to the Space"""
-    file_size = file_path.stat().st_size / (1024 * 1024)  # Size in MB
-    print(f"Uploading {file_path.name} ({file_size:.2f} MB)...")
-    
-    # Create the commit operation
-    operation = CommitOperationAdd(
-        path_in_repo=str(repo_path),
-        path_or_fileobj=str(file_path)
-    )
-    
-    # Upload the file
-    api.create_commit(
-        repo_id=Settings.HF_SPACE,
-        repo_type="space",
-        operations=[operation],
-        commit_message=f"Upload {file_path.name}"  # Removed async flag
-    )
-    
-    # Wait for file to appear in repository
-    if file_size > 100:  # Only wait for files larger than 100MB
-        if wait_for_file_upload(api, file_path, repo_path):
-            print(f"Successfully uploaded and verified {file_path.name}")
-        else:
-            raise RuntimeError(f"Failed to verify upload of {file_path.name}")
-    else:
-        print(f"Successfully uploaded {file_path.name}")
-
-
 def main():
     """Upload embeddings directory to HuggingFace Space"""
     if not Settings.HF_TOKEN:
@@ -105,25 +52,14 @@ def main():
     
     print(f"Uploading to Space: {Settings.HF_SPACE}...")
     try:
-        # Upload each file individually
-        for file_path in Settings.EMBEDDINGS_DIR.rglob('*'):
-            if file_path.is_file() and not any(
-                pattern in str(file_path) 
-                for pattern in ['*.pyc', '__pycache__', '.DS_Store']
-            ):
-                # Calculate relative path for the repo
-                rel_path = file_path.relative_to(Settings.EMBEDDINGS_DIR)
-                repo_path = f"data/processed/embeddings/{rel_path}"
-                
-                try:
-                    # Upload the file
-                    upload_large_file(api, file_path, repo_path)
-                except Exception as e:
-                    print(f"Error uploading {file_path.name}: {str(e)}")
-                    if "storage" in str(e).lower():
-                        print(f"\nFile size: {file_path.stat().st_size / (1024*1024):.2f} MB")
-                    raise
-        
+        api.upload_folder(
+            folder_path=str(Settings.EMBEDDINGS_DIR),
+            repo_id=Settings.HF_SPACE,
+            repo_type="space",
+            path_in_repo="data/processed/embeddings",
+            ignore_patterns=["*.pyc", "__pycache__", ".DS_Store"],
+            commit_message="Upload embeddings to Space storage"
+        )
         print("Upload complete!")
     except Exception as e:
         print(f"Error during upload: {str(e)}")