upload_to_hf.py

import os
from huggingface_hub import HfApi, create_repo

token = os.environ.get("HF_TOKEN")
if not token:
    raise ValueError("HF_TOKEN not found in environment.")

api = HfApi(token=token)
repo_id = "rafmacalaba/wbg_annotation_data"

print(f"Creating dataset repository: {repo_id}")
try:
    create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True, token=token)
    print("Repository created or already exists.")
except Exception as e:
    print(f"Error creating repo: {e}")

folder_path = "annotation_data"
print(f"Uploading {folder_path} to {repo_id}...")

api.upload_folder(
    folder_path=folder_path,
    repo_id=repo_id,
    repo_type="dataset",
    path_in_repo="annotation_data"
)

print("Upload complete! Data is now available at https://huggingface.co/datasets/" + repo_id)