bldr / wheelmaker.py
lainlives's picture
Upload folder using huggingface_hub
cc2f72b verified
import os
import tempfile
import subprocess
from venv import create
import shutil
import sys
from pathlib import Path
from huggingface_hub import HfApi, create_repo
# --- Configuration ---
GITHUB_REPO_URL = os.environ.get("GITHUB_REPO_URL")
REPO_LIST = os.environ.get("GITHUB_REPO_LIST", '').split()
HF_REPO_ID = os.environ.get("HF_REPO_ID")
HF_TOKEN = os.environ.get("HF_TOKEN")
PRIVATE = os.environ.get("PRIVATE", "false")
REPO_TYPE = os.environ.get("REPO_TYPE", "model")
create(".env", system_site_packages=True, with_pip=True, clear=True)
python_executable = os.path.join(".env", "bin", "python")
PYTHON_VERSIONS = ["python3.10", "python3.11", "python3.12", "python3.13", "python3.14", "python3.15"]
# Setup directories
WHEEL_OUTPUT_DIR = Path(tempfile.mkdtemp())
def clean_env(work_dir):
"""Cleans up previous build artifacts."""
if work_dir.exists():
shutil.rmtree(work_dir)
if WHEEL_OUTPUT_DIR.exists():
shutil.rmtree(WHEEL_OUTPUT_DIR)
def clone_repo(work_dir, repo):
"""Clones the GitHub repository."""
print(f"⬇️ Cloning {repo}...")
subprocess.check_call(["git", "clone", repo, str(work_dir)])
def build_wheels(work_dir):
"""Builds wheels for each specified Python version."""
# repo_name = repo.split("/")[-1].replace(".git", "")
project_path = work_dir
# Check if git clone created a subdirectory (standard behavior)
# If the work_dir contains the .git directly, use it, otherwise find the folder
cloned_folders = [f for f in work_dir.iterdir() if f.is_dir()]
if len(cloned_folders) == 1:
project_path = cloned_folders[0]
print(f"πŸ”¨ Building in: {project_path}")
for python_cmd in PYTHON_VERSIONS:
print(f"βš™οΈ Building for {python_cmd}...")
try:
# Ensure 'build' is installed for that specific python version
subprocess.check_call(
[python_cmd, "-m", "pip", "install", "build"],
stdout=subprocess.DEVNULL
)
# Run the build command
# We output to our main wheel directory to collect them all in one place
subprocess.check_call(
[python_cmd, "-m", "build", "--wheel", "--outdir", str(WHEEL_OUTPUT_DIR.absolute())],
cwd=project_path
)
except subprocess.CalledProcessError as e:
print(f"❌ Failed to build for {python_cmd}. Ensure it is installed and in PATH.")
print(e)
except FileNotFoundError:
print(f"❌ Executable {python_cmd} not found on system.")
def upload_to_hf():
"""Uploads the collected wheels to Hugging Face."""
if not any(WHEEL_OUTPUT_DIR.iterdir()):
print("⚠️ No wheels found to upload. Exiting.")
return
print(f"πŸš€ Uploading wheels to Hugging Face Repo: {HF_REPO_ID}...")
api = HfApi(token=HF_TOKEN)
try:
create_repo(repo_id=HF_REPO_ID, private=PRIVATE, repo_type=REPO_TYPE)
api.upload_folder(
folder_path=WHEEL_OUTPUT_DIR,
repo_id=HF_REPO_ID,
repo_type=REPO_TYPE,
commit_message=f"Update wheels for Python {', '.join([v.replace('python', '') for v in PYTHON_VERSIONS])}"
)
print("βœ… Upload complete!")
except Exception as e:
print(f"❌ Upload failed: {e}")
def build_all(repo_list):
for repo in repo_list:
work_dir = Path(tempfile.mkdtemp())
try:
clean_env(work_dir)
clone_repo(work_dir, repo)
build_wheels(work_dir)
upload_to_hf()
shutil.rmtree(work_dir)
finally:
# Optional: Cleanup after running
pass
def main():
if not HF_TOKEN:
print("❌ Error: HF_TOKEN environment variable not set.")
sys.exit(1)
build_all(REPO_LIST)
if __name__ == "__main__":
main()
#
#
#
#
#
#
#