from huggingface_hub import HfApi

api = HfApi()
print("🚀 Starting large folder upload... This will safely chunk your 9GB of data.")

api.upload_large_folder(
    repo_id="CALLMEshahryar/new_cgc_data",
    folder_path="/home/pourtaheri/new_cgc_data/",
    repo_type="model",
    ignore_patterns=["raw_data/*"]
)

print("✅ Upload successfully completed!")