sync_dataset.py

from argparse import ArgumentParser
from huggingface_hub import HfApi


def upload_videos(api, upload_dir):
    api.upload_folder(
        folder_path=upload_dir,
        repo_id="zirui-wang/survey-videos",
        repo_type="dataset",
    )


def delete_response(api):
    api.delete_folder("response", "zirui-wang/survey-videos", repo_type="dataset")


def delete_optional_feedback(api):
    api.delete_folder("optional_feedback", "zirui-wang/survey-videos", repo_type="dataset")


def main():
    parser = ArgumentParser()
    parser.add_argument(
        "--upload_dir",
        type=str,
        default="~/projects/mview/storage/data/video_diffusion/text-to-video-debug-anonymised",
    )
    parser.add_argument("--upload_videos", type=eval, default=False, choices=[True, False])
    parser.add_argument("--delete_response", type=eval, default=False, choices=[True, False])
    parser.add_argument(
        "--delete_optional_feedback", type=eval, default=False, choices=[True, False]
    )
    args = parser.parse_args()

    api = HfApi()
    if args.upload_videos:
        upload_videos(api, args.upload_dir)

    if args.delete_response:
        answer = input("Are you sure you want to delete the response folder? (y/n)")
        if answer == "y":
            try:
                delete_response(api)
            except Exception as e:
                print(e)

    if args.delete_optional_feedback:
        answer = input("Are you sure you want to delete the optional_feedback folder? (y/n)")
        if answer == "y":
            try:
                delete_optional_feedback(api)
            except Exception as e:
                print(e)

    api.super_squash_history("zirui-wang/survey-videos", repo_type="dataset")


if __name__ == "__main__":
    main()