Initial Commit

app.py

@@ -0,0 +1,71 @@
+import os
+import time
+import shutil
+import pandas as pd
+import gradio as gr
+from vidfetch import youtube_dl_install_helper, push_to_hf
+from panda70m_downloader import download_video_links, download_videos_by_csv
+
+
+SAVE_CSV_DIR = "panda70m_csv"
+SAVE_VIDEOS_DIR = "panda70m_videos"
+
+def handle(
+    hf_token: str, 
+    filename: str, 
+):
+    try:
+        import youtube_dl
+    except:
+        youtube_dl_install_helper(hf_token=hf_token)
+        import youtube_dl
+    
+    download_video_links(hf_token=hf_token, filename=filename, save_dir=SAVE_CSV_DIR)
+    
+    # devide .csv to 100 files and download
+    csv_path = os.path.join(SAVE_CSV_DIR, filename)
+    data = pd.read_csv(csv_path)
+    for idx in range(len(data) // 100):
+        begin_idx = idx * 100
+        end_idx = idx * 100 + 100
+        part_data = data[begin_idx : end_idx]
+        part_filename = filename.replace(".csv", "") + "_{:06d}_{:06d}.csv".format(begin_idx, end_idx)
+        targz_filename = part_filename.replace(".csv", ".tar.gz")
+        part_save_path = os.path.join(SAVE_CSV_DIR, part_filename)
+        part_data.to_csv(part_save_path)
+        download_videos_by_csv(
+            csv_file_path=part_save_path,
+            save_dir=SAVE_VIDEOS_DIR,
+            targz_filename=targz_filename
+        )
+        push_to_hf(
+            hf_token=hf_token,
+            hf_repo_id="OpenVideo/Panda-70M-raw",
+            file_path=os.path.join(SAVE_VIDEOS_DIR, targz_filename),
+            path_in_repo=targz_filename
+        )
+        shutil.rmtree(SAVE_VIDEOS_DIR)
+        
+
+with gr.Blocks() as demo:
+    gr.Markdown(
+        '''
+        Panda70M-Downloader
+        '''
+    )
+    hf_token = gr.Textbox(label="HuggingFace Token")
+    filename = gr.Textbox(label="csv name")
+
+    with gr.Row():
+        button = gr.Button("Submit", variant="primary")
+        clear = gr.Button("Clear")
+
+    button.click(
+        handle, 
+        [hf_token, filename], 
+        outputs=None
+    )
+
+
+if __name__ == "__main__":
+    demo.launch(debug = True)

panda70m_downloader.py

@@ -0,0 +1,85 @@
+import os
+import shutil
+import pandas as pd
+from vidfetch import compress_folder, pull_from_hf
+
+
+def download_video_links(hf_token: str, filename: str, save_dir: str):
+    # check save dir
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+    
+    # download
+    pull_from_hf(
+        hf_token=hf_token,
+        hf_repo_id="OpenVideo/Panda-70M-Original-Links",
+        filename=filename,
+        save_dir=save_dir
+    )
+    
+
+def download_videos_by_csv(
+    csv_file_path: str,
+    save_dir: str, 
+    targz_filename: str,
+):
+    try:
+        import youtube_dl
+    except:
+        raise ModuleNotFoundError(
+            "youtube_dl missed, please install it by ``vidfetch.package.youtube.youtube_dl_install_helper``"
+        )
+    # path/dir
+    folder_name = targz_filename.replace(".tar.gz", "")
+    download_videos_dir = os.path.join(save_dir, folder_name, "download_raw")    
+    log_path = os.path.join(download_videos_dir, "log.txt")
+    targz_path = os.path.join(save_dir, targz_filename)
+    
+    # make dirs
+    if not os.path.exists(download_videos_dir):
+        os.makedirs(download_videos_dir)
+    
+    # read from csv
+    csv_filename = os.path.basename(csv_file_path)
+    shutil.copy(src=csv_file_path, dst=os.path.join(download_videos_dir, csv_filename))
+    data = pd.read_csv(csv_file_path)
+    links = data["url"].tolist()
+    videos_id = data["videoID"].to_list()    
+
+    failed_links = [] # record failed links
+    for link, video_id in zip(links, videos_id):
+        # check if downloaded
+        video_save_path = os.path.join(download_videos_dir, video_id[1:]+".mp4")
+        if os.path.exists(video_save_path):
+            continue
+        
+        # download
+        ydl_opts = {
+            'format': 'best',
+            'quiet': False,
+            'outtmpl': os.path.join(download_videos_dir, video_id[1:]+".mp4"),
+        }
+        with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+            try:
+                ydl.download([link])
+            except:
+                failed_links.append(link)
+
+    # delete videos larger than 100MB
+    video_files = os.listdir(download_videos_dir)
+    delete_videos = []
+    for file in video_files:
+        file_path = os.path.join(download_videos_dir, file)
+        file_size_mb = os.path.getsize(file_path) / (1024 * 1024)  # Convert to megabytes
+        if file_size_mb > 500:
+            delete_videos.append(file_path)
+            os.remove(file_path)
+    
+    # Write  to log file
+    with open(log_path, 'w') as file:
+        file.write('Fail to download\n')
+        file.write('\n'.join(failed_links))
+        file.write('Delete videos larger than 500MB\n')
+        file.write('\n'.join(failed_links))
+    
+    compress_folder(download_videos_dir, targz_path)

requirements.txt

@@ -0,0 +1,3 @@
+vidfetch
+gradio
+pandas