scripts/upload_to_huggingface.py

#!/usr/bin/env python3
"""
Upload MemGen to HuggingFace Hub

Usage:
    python scripts/upload_to_huggingface.py --repo_id LCZZZZ/model111
"""

import os
import argparse
from pathlib import Path
from huggingface_hub import HfApi, create_repo, upload_folder
import logging

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')


def create_gitignore(base_path: str):
    """Create a .gitignore file to exclude unnecessary files."""
    gitignore_content = """
# Python cache
__pycache__/
*.py[cod]
*$py.class
*.so
.Python

# Virtual environments
venv/
env/
ENV/

# IDE
.vscode/
.idea/
*.swp
*.swo

# Large model files (if any)
*.bin
*.safetensors
*.pth
*.pt
*.ckpt

# Logs and outputs (optional - comment out if you want to upload)
test_output/
wandb/
*.log

# Backup files
*_backup_original/
*.backup
*.backup_no_letters

# Data caches
dataset/*/cache/
.cache/

# System files
.DS_Store
Thumbs.db
"""
    
    gitignore_path = os.path.join(base_path, ".gitignore")
    if not os.path.exists(gitignore_path):
        with open(gitignore_path, 'w') as f:
            f.write(gitignore_content)
        logging.info(f"Created .gitignore at {gitignore_path}")
    else:
        logging.info(f".gitignore already exists at {gitignore_path}")


def upload_to_hf(repo_id: str, folder_path: str, create_if_not_exists: bool = True):
    """
    Upload a folder to HuggingFace Hub.
    
    Args:
        repo_id: HuggingFace repository ID (e.g., "username/repo_name")
        folder_path: Local folder path to upload
        create_if_not_exists: Whether to create the repo if it doesn't exist
    """
    api = HfApi()
    
    # Check if repo exists, create if needed
    try:
        api.repo_info(repo_id=repo_id, repo_type="model")
        logging.info(f"Repository {repo_id} already exists")
    except Exception as e:
        if create_if_not_exists:
            logging.info(f"Repository {repo_id} not found, creating...")
            create_repo(repo_id=repo_id, repo_type="model", exist_ok=True)
            logging.info(f"Repository {repo_id} created successfully")
        else:
            raise Exception(f"Repository {repo_id} does not exist and create_if_not_exists=False")
    
    # Create .gitignore to exclude unnecessary files
    create_gitignore(folder_path)
    
    # Upload folder
    logging.info(f"Uploading {folder_path} to {repo_id}...")
    logging.info("This may take a while depending on the size...")
    
    try:
        result = upload_folder(
            repo_id=repo_id,
            folder_path=folder_path,
            repo_type="model",
            commit_message="Upload MemGen code and data",
            ignore_patterns=[
                "*.pyc",
                "__pycache__",
                ".git",
                "*.log",
                "test_output/*",
                "wandb/*",
                "*_backup_original/*",
                "*.backup",
                "*.backup_no_letters",
                ".cache/*",
            ]
        )
        
        logging.info(f"✓ Upload completed successfully!")
        logging.info(f"Repository URL: https://huggingface.co/{repo_id}")
        return result
        
    except Exception as e:
        logging.error(f"Upload failed: {e}")
        raise


def main():
    parser = argparse.ArgumentParser(description="Upload MemGen to HuggingFace Hub")
    parser.add_argument(
        "--repo_id",
        type=str,
        default="LCZZZZ/model111",
        help="HuggingFace repository ID (e.g., 'username/repo_name')"
    )
    parser.add_argument(
        "--folder_path",
        type=str,
        default="/root/CVPR/MemGen",
        help="Local folder path to upload"
    )
    parser.add_argument(
        "--no-create",
        action="store_true",
        help="Don't create repository if it doesn't exist"
    )
    
    args = parser.parse_args()
    
    if not os.path.exists(args.folder_path):
        logging.error(f"Folder not found: {args.folder_path}")
        return
    
    print("=" * 80)
    print("Upload to HuggingFace Hub")
    print("=" * 80)
    print(f"Repository: {args.repo_id}")
    print(f"Folder: {args.folder_path}")
    print("=" * 80)
    print("\nStarting upload...")
    
    upload_to_hf(
        repo_id=args.repo_id,
        folder_path=args.folder_path,
        create_if_not_exists=not args.no_create
    )


if __name__ == "__main__":
    main()