Spaces:
Sleeping
Sleeping
File size: 4,219 Bytes
20cfecf b1279f9 20cfecf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 |
#!/usr/bin/env python3
"""
Upload pre-compiled CUDA kernels to Hugging Face Dataset.
This avoids git push issues with binary files on Hugging Face Spaces.
Usage:
python upload_kernels_to_dataset.py
The kernels will be uploaded to: huggingface.co/datasets/oliau/styleforge-kernels
"""
import sys
from pathlib import Path
try:
from huggingface_hub import HfApi, login
except ImportError:
print("ERROR: huggingface_hub not installed.")
print("Install with: pip install huggingface_hub")
sys.exit(1)
def upload_kernels():
"""Upload pre-compiled kernels to Hugging Face dataset."""
# Configuration
DATASET_ID = "oliau/styleforge-kernels"
PREBUILT_DIR = Path("kernels/prebuilt")
print("=" * 60)
print("StyleForge Kernel Uploader")
print("=" * 60)
print()
# Check if prebuilt directory exists
if not PREBUILT_DIR.exists():
print(f"ERROR: Prebuilt directory not found: {PREBUILT_DIR}")
print("Run compile_kernels.py first to generate the kernels.")
sys.exit(1)
# Find all kernel files
kernel_files = list(PREBUILT_DIR.glob("*.so")) + list(PREBUILT_DIR.glob("*.pyd"))
if not kernel_files:
print(f"ERROR: No kernel files found in {PREBUILT_DIR}")
print("Expected .so or .pyd files.")
sys.exit(1)
print(f"Found {len(kernel_files)} kernel file(s):")
for f in kernel_files:
size_kb = f.stat().st_size / 1024
print(f" - {f.name} ({size_kb:.1f} KB)")
print()
# Initialize HF API
api = HfApi()
# Check if user is logged in
try:
whoami = api.whoami()
print(f"Logged in as: {whoami.get('name', whoami.get('user', 'unknown'))}")
except Exception:
print("Not logged in to Hugging Face.")
print("Please run: huggingface-cli login")
print("Or set HF_TOKEN environment variable.")
sys.exit(1)
print()
print(f"Uploading to dataset: {DATASET_ID}")
print()
# Create dataset if it doesn't exist
try:
repo_info = api.repo_info(DATASET_ID, repo_type="dataset")
print(f"Dataset exists: {DATASET_ID}")
except Exception:
print(f"Creating new dataset: {DATASET_ID}")
api.create_repo(
repo_id=DATASET_ID.split("/")[1],
repo_type="dataset",
private=False,
exist_ok=True
)
print(f"Dataset created: {DATASET_ID}")
# Create README for the dataset
readme_content = """---
title: StyleForge CUDA Kernels
license: mit
tags:
- cuda
- neural-style-transfer
- styleforge
---
# StyleForge Pre-compiled CUDA Kernels
This repository contains pre-compiled CUDA kernels for the StyleForge neural style transfer project.
## Files
"""
for f in kernel_files:
readme_content += f"- `{f.name}`\n"
readme_content += """
## Usage
These kernels are automatically downloaded by StyleForge when running on Hugging Face Spaces.
## Compilation
Kernels are compiled for multiple GPU architectures:
- sm_70 (V100)
- sm_75 (T4)
- sm_80 (A100)
For local compilation, see `compile_kernels.py` in the main repository.
"""
# Upload files
print("Uploading files...")
# Upload README
api.upload_file(
path_or_fileobj=readme_content.encode(),
path_in_repo="README.md",
repo_id=DATASET_ID,
repo_type="dataset",
commit_message="Add dataset README"
)
print(" Uploaded: README.md")
# Upload kernel files
for kernel_file in kernel_files:
print(f" Uploading {kernel_file.name}...", end=" ", flush=True)
api.upload_file(
path_or_fileobj=str(kernel_file),
path_in_repo=kernel_file.name,
repo_id=DATASET_ID,
repo_type="dataset",
commit_message=f"Add {kernel_file.name}"
)
print("✓")
print()
print("=" * 60)
print("Upload complete!")
print("=" * 60)
print()
print(f"Dataset URL: https://huggingface.co/datasets/{DATASET_ID}")
print()
print("The kernels will be automatically downloaded by StyleForge")
print("when running on Hugging Face Spaces.")
if __name__ == "__main__":
upload_kernels()
|