update

toolbox/hf_netdisk/netdisk/netdisk_client.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 import argparse
+from pathlib import Path
+import shutil
 from typing import List
 
 from huggingface_hub.hf_api import CommitInfo
@@ -34,10 +36,22 @@ class NetdiskClient(RepoClient):
         result: CommitInfo = self.hf_api.delete_file(
             path_in_repo=path_in_repo,
             repo_id=repo_id,
-            repo_type="dataset"
+            repo_type="dataset",
         )
         return result
 
+    def download_file(self, remote_file: str, local_file: str, repo_id: str, cache_dir: str = None):
+        file_path: str = self.hf_api.hf_hub_download(
+            filename=remote_file,
+            repo_id=repo_id,
+            repo_type="dataset",
+            cache_dir=cache_dir
+        )
+        local_file = Path(local_file)
+        local_file.parent.mkdir(parents=True, exist_ok=True)
+        shutil.move(file_path, local_file.as_posix())
+        return local_file
+
 
 def get_args():
     parser = argparse.ArgumentParser()
@@ -86,6 +100,13 @@ def main():
     )
     print(result)
 
+    # result = client.download_file(
+    #     remote_file="xianyu/customers/陈杰森/20251226/焦虑个锤子(20251225_165801直播).mp4",
+    #     local_file="temp.mp4",
+    #     repo_id="tianxing1994/hf_netdisk",
+    # )
+    # print(result)
+
     result = client.delete_file(
         path_in_repo=args.tgt_file,
         repo_id="tianxing1994/hf_netdisk",

toolbox/youtube_spider/video/video_download.py

@@ -54,11 +54,17 @@ class YoutubeVideoDownloadSpider(YoutubeSpiderClient):
         target_file = Path(target_file)
         target_file.parent.mkdir(parents=True, exist_ok=True)
 
+        # ydl_opts = {
+        #     'outtmpl': target_file.as_posix(),
+        #     'format': 'bestvideo+bestaudio/best',  # 下载最佳视频+音频
+        #     'merge_output_format': 'mp4',  # 合并成 mp4
+        #     'noplaylist': True,  # 只下载单个视频，不下载播放列表
+        # }
         ydl_opts = {
             'outtmpl': target_file.as_posix(),
-            'format': 'bestvideo+bestaudio/best',  # 下载最佳视频+音频
-            'merge_output_format': 'mp4',  # 合并成 mp4
-            'noplaylist': True,  # 只下载单个视频，不下载播放列表
+            "format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
+            'merge_output_format': 'mp4',
+            'noplaylist': True,
         }
 
         self.delay_before_download()
@@ -78,5 +84,44 @@ def main():
     return
 
 
+def main2():
+    import json
+    from project_settings import project_path
+
+    client = YoutubeVideoDownloadSpider()
+
+    filename = project_path / "temp/filename.jsonl"
+    target_dir = project_path / "temp/陈杰森"
+    target_dir.mkdir(parents=True, exist_ok=True)
+
+    # finished
+    finished_set = set()
+    for video_file in target_dir.glob("*.mp4"):
+        finished_set.add(video_file.stem)
+    print(f"finished_count: {len(finished_set)}")
+
+    rows = list()
+    with open(filename, "r", encoding="utf-8") as f:
+        for row in f:
+            row = json.loads(row)
+            rows.append(row)
+
+    for row in list(reversed(rows)):
+        video_id = row["video_id"]
+        title = row["title"]
+        if title in finished_set:
+            continue
+
+        print(f"download; video_id: {video_id}, title: {title}")
+        client.download_by_video_id_by_yt_dlp(
+            video_id=video_id,
+            target_file=(target_dir / f"{title}.mp4").as_posix()
+        )
+        print(f"sleeping ...")
+        time.sleep(1800)
+
+    return
+
+
 if __name__ == "__main__":
-    main()
+    main2()

toolbox/youtube_spider/video/video_list.py

@@ -170,8 +170,11 @@ class YoutubeVideoListSpider(YoutubeSpiderClient):
             })
 
         # print(json.dumps(continuation_grid_renderer, ensure_ascii=False, indent=4))
-        continuation_item_renderer = continuation_grid_renderer["continuationItemRenderer"]
-        continuation_token = continuation_item_renderer["continuationEndpoint"]["continuationCommand"]["token"]
+
+        continuation_token = None
+        continuation_item_renderer = continuation_grid_renderer.get("continuationItemRenderer")
+        if continuation_item_renderer is not None:
+            continuation_token = continuation_item_renderer["continuationEndpoint"]["continuationCommand"]["token"]
         return video_list, continuation_token
 
 
@@ -192,5 +195,54 @@ def main():
     return
 
 
+def main2():
+    import re
+
+    client = YoutubeVideoListSpider()
+
+
+    pattern = r"(\d{8})_(\d{6})"
+
+    filename = "filename.jsonl"
+
+    with open(filename, "a+", encoding="utf-8") as f:
+        video_list, continuation_token = client.get_front_page_video_list_pretty("JasonBear131")
+        for video in video_list:
+            video_id = video["video_id"]
+            title = video["title"]
+            match = re.search(pattern, string=title, flags=re.IGNORECASE)
+            if match is None:
+                continue
+            print(title)
+
+            row = {
+                "video_id": video_id,
+                "title": title,
+            }
+            row = json.dumps(row, ensure_ascii=False)
+            f.write(f"{row}\n")
+            f.flush()
+
+        for i in range(1000):
+            video_list, continuation_token = client.get_continuation_page_video_list_pretty(continuation_token)
+            for video in video_list:
+                video_id = video["video_id"]
+                title = video["title"]
+                match = re.search(pattern, string=title, flags=re.IGNORECASE)
+                if match is None:
+                    continue
+                print(title)
+
+                row = {
+                    "video_id": video_id,
+                    "title": title,
+                }
+                row = json.dumps(row, ensure_ascii=False)
+                f.write(f"{row}\n")
+                f.flush()
+
+    return
+
+
 if __name__ == "__main__":
-    main()
+    main2()