update

data/porter_tasks/{porter_task_chenjieshen_douyin_live_record_to_baidu_netdisk.json → porter_task_chenjieshen_douyin_live_record_to_hf_netdisk.json}

@@ -1,187 +1,177 @@
 [
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "陈杰森资本NewBoombap",
     "room_id": "57544215290",
     "sec_user_id": "MS4wLjABAAAATGoBrO7yiJ3q9go4fxq9JXjrnP1bFpdkgKckC1IpfXA_vrjSmL9ZtjmTju8ApwbT",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "老陈的退路",
     "room_id": "330025930592",
     "sec_user_id": "MS4wLjABAAAAzjLy7Dd3Q-UiMGdkwymLhDIL1aFJtruvCUqrwmph9n8Zb55E4y9XH2WEFaLC1ZBl",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "老陈come_back",
     "room_id": "78835697536",
     "sec_user_id": "MS4wLjABAAAAj3qTpPkes0t5az2nXd1oaag5TpOJeNh27pemT17WUVXHGUZKZfjqc3fFc8arL6V1",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "清源第一帅",
     "room_id": "654177813521",
     "sec_user_id": "MS4wLjABAAAAtwkRNs5Krtm5cBhKzn60poflAFWdnYdwMf0I0M27rOGvNn2l16DfUH5vE3WL-ypz",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "清源第二帅",
     "room_id": "831568261978",
     "sec_user_id": "MS4wLjABAAAALbRezHNzlaRYhk9QE-5ajImgcvfJGq9koAlD0O8VQwiC-N8Bqx2WDW07wLee2Bwd",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "清源第三帅",
     "room_id": "26569028431",
     "sec_user_id": "MS4wLjABAAAA60pV9JuZNTZhQJeSI_AeaRUnKXs--TGPZEAtsgihwKnir46ncFVtLSO79JDnclkM",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "清源第四帅",
     "room_id": "91434255754",
     "sec_user_id": "MS4wLjABAAAAakYu9rpF6vk50XbiKFnxzuXVv6fhVK5O3tr0gxjTJq2Nw5HfMplPv6ce6AhXxReG",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "清源第五帅",
     "room_id": "469245033355",
     "sec_user_id": "MS4wLjABAAAA62G-uDXdlIPnk44P9gXhQQo1UmOCgZ08QWOP_-t4PjI",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "清源人工智能研究院",
     "room_id": "81728900292",
     "sec_user_id": "MS4wLjABAAAAe_Hf6esmopqIMG7OhsMlNESTiqP1ot7lmGzK6Cu3pJSSGTOiHlI4cA33OPWg56Ql",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "廖勇-清源政策研究院AIGC",
     "room_id": "gubenqingyuan",
     "sec_user_id": "MS4wLjABAAAA9ne6zoCLjnzpCRR1GhPVBUq2_7wIJO4GyZXlyW016yw",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "小熊Bella与老爸",
     "room_id": "139751520143",
     "sec_user_id": "MS4wLjABAAAA49QFP6YhorLIIX9M-FiZeKxmqhqXlttluSsZeaxvxzU",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "清源之虎",
     "room_id": "998621457719",
     "sec_user_id": "MS4wLjABAAAAWYFqu46IMCnfwgSjBK35RnQcKOYhqVX3YcP4rotpHkdxJn5JD59REMz5fwlccp2h",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "老陈真是好人",
     "room_id": "599130203190",
     "sec_user_id": "MS4wLjABAAAA3ReegyjpprzeurqHUl8yLBkTlQ7L4TfoyjpDXeiy1RvCu7OmqWljOLmWgibTjs8y",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "douyin_live_record",
     "room_name": "老陈小帮手",
     "room_id": "738682070097",
     "sec_user_id": "MS4wLjABAAAAtzZsG1mKtZvgE2j58ZB7FDRP1gzsHXuQDIN15bltkLM",
     "key_of_credentials": "douyin_wentao_credentials",
     "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
-  },
-  {
-    "enable": false,
-    "type": "douyin_live_record",
-    "room_name": "焦虑个锤子",
-    "room_id": "78933304321",
-    "sec_user_id": "MS4wLjABAAAAfr5os1dB5zdm36GK5nHVQkWReaELMueuYqliVDBo64Y",
-    "key_of_credentials": "douyin_wentao_credentials",
-    "check_interval": 10,
-    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森",
-    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json"
+    "output_video_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森",
+    "output_video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json"
   },
   {
-    "enable": false,
+    "enable": true,
     "type": "video_format_convert",
-    "task_name": "陈杰森-百度云盘-格式转换",
-    "video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/file_info.json",
-    "target_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/to_baidu_netdisk",
+    "task_name": "陈杰森",
+    "video_info_file": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/file_info.json",
+    "target_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/ready_to_hf_netdisk",
     "check_interval": 10,
     "remove_after_upload": true,
     "format_pairs": [["flv", "mp4"]]
   },
   {
     "enable": true,
-    "type": "file_to_baidu_netdisk",
+    "type": "file_to_hf_netdisk",
     "task_name": "陈杰森",
-    "src_dir": "data/tasks/chenjieshen_douyin_live_record_to_baidu_netdisk/live_record/陈杰森/to_baidu_netdisk",
-    "tgt_dir": "xianyu/customers/{task_name}/{date_str}",
+    "src_dir": "data/tasks/chenjieshen_douyin_live_record_to_hf_netdisk/live_record/陈杰森/ready_to_hf_netdisk",
+    "tgt_dir": "xianyu/customers/陈杰森/{date_str}",
+    "repo_id": "tianxing1994/hf_netdisk",
     "check_interval": 10,
-    "key_of_credentials": "baidu_netdisk_honeytian_credentials",
+    "key_of_credentials": "hf_miyuki_write_token",
     "remove_after_upload": true,
     "exclude_files": "file_info.json"
   }

toolbox/hf_netdisk/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == "__main__":
+    pass

toolbox/hf_netdisk/hf_netdisk_client.py

@@ -0,0 +1,77 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import logging
+import shutil
+import tempfile
+from pathlib import Path
+
+import requests
+from huggingface_hub import HfApi, login
+
+from project_settings import project_path, environment
+from toolbox.design_patterns.singleton import ParamsSingleton
+
+
+logger = logging.getLogger("toolbox")
+
+
+class HfNetdiskClient(ParamsSingleton):
+
+    def __init__(self, hf_token: str = None):
+        if not self._initialized:
+            self.hf_token = hf_token
+
+            self.hf_api = HfApi(token=hf_token)
+
+            self._initialized = True
+
+    def check_login(self):
+        flag = False
+        try:
+            user_info = self.hf_api.whoami()
+            user_info = json.dumps(user_info, ensure_ascii=False)
+            flag = True
+        except requests.exceptions.HTTPError as error:
+            user_info = f"whoami failed; error type: {type(error)}, error text: {str(error)}"
+        except Exception as error:
+            user_info = f"whoami failed; error type: {type(error)}, error text: {str(error)}"
+
+        logger.info(f"login flag: {flag}, user_info: {user_info}")
+        return flag
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--key_of_credentials",
+        default="hf_miyuki_write_token",
+        type=str,
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    import log
+    from project_settings import environment, project_path, log_directory, time_zone_info
+
+    log.setup_size_rotating(log_directory=log_directory, tz_info=time_zone_info)
+
+    hf_token = environment.get(key=args.key_of_credentials)
+
+    client = HfNetdiskClient(
+        hf_token=hf_token,
+    )
+
+    client.check_login()
+
+
+    return
+
+
+if __name__ == "__main__":
+    main()

toolbox/hf_netdisk/netdisk/netdisk_client.py

@@ -0,0 +1,98 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from typing import List
+
+from huggingface_hub.hf_api import CommitInfo
+
+from project_settings import project_path
+from toolbox.hf_netdisk.netdisk.repo_client import RepoClient
+
+
+class NetdiskClient(RepoClient):
+    def __init__(self, hf_token: str):
+        super(NetdiskClient, self).__init__(hf_token=hf_token)
+
+    def upload_file(self, src_file, tgt_file, repo_id: str) -> str:
+        self.make_sure_repo(repo_id)
+        tgt_file_url = self.hf_api.upload_file(
+            path_or_fileobj=src_file,
+            path_in_repo=tgt_file,
+            repo_id=repo_id,
+            repo_type="dataset"
+        )
+        return tgt_file_url
+
+    def query_files(self, repo_id: str) -> List[str]:
+        result: List[str] = self.hf_api.list_repo_files(
+            repo_id=repo_id,
+            repo_type="dataset"
+        )
+        return result
+
+    def delete_file(self, path_in_repo: str, repo_id: str) -> str:
+        result: CommitInfo = self.hf_api.delete_file(
+            path_in_repo=path_in_repo,
+            repo_id=repo_id,
+            repo_type="dataset"
+        )
+        return result
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--key_of_credentials",
+        default="hf_miyuki_write_token",
+        type=str,
+    )
+    parser.add_argument(
+        "--src_file",
+        default=(project_path / "README.md").as_posix(),
+        type=str,
+    )
+    parser.add_argument(
+        "--tgt_file",
+        default="video_platform/upload/README.md",
+        type=str,
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    import log
+    from project_settings import environment, project_path, log_directory, time_zone_info
+
+    log.setup_size_rotating(log_directory=log_directory, tz_info=time_zone_info)
+
+    hf_token = environment.get(key=args.key_of_credentials)
+
+    client = NetdiskClient(
+        hf_token=hf_token,
+    )
+
+    result = client.upload_file(
+        repo_id="tianxing1994/hf_netdisk",
+        src_file=args.src_file,
+        tgt_file=args.tgt_file,
+    )
+    print(result)
+
+    result = client.query_files(
+        repo_id="tianxing1994/hf_netdisk",
+    )
+    print(result)
+
+    result = client.delete_file(
+        path_in_repo=args.tgt_file,
+        repo_id="tianxing1994/hf_netdisk",
+    )
+    print(result)
+    return
+
+
+if __name__ == "__main__":
+    main()

toolbox/hf_netdisk/netdisk/repo_client.py

@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+
+from toolbox.hf_netdisk.hf_netdisk_client import HfNetdiskClient
+
+
+class RepoClient(HfNetdiskClient):
+    def __init__(self, hf_token: str):
+        super(RepoClient, self).__init__(hf_token=hf_token)
+
+    def create_repo(self, repo_id: str) -> str:
+        self.hf_api.create_repo(
+            repo_id=repo_id,
+            repo_type="dataset",
+            private=False,
+            exist_ok=True
+        )
+        return repo_id
+
+    def delete_repo(self, repo_id: str) -> str:
+        self.hf_api.delete_repo(
+            repo_id=repo_id,
+            repo_type="dataset",
+            missing_ok=True
+        )
+        return repo_id
+
+    def check_repo(self, repo_id: str) -> bool:
+        exists: bool = self.hf_api.repo_exists(
+            repo_id=repo_id,
+            repo_type="dataset",
+        )
+        return exists
+
+    def make_sure_repo(self, repo_id: str) -> bool:
+        exists: bool = self.hf_api.repo_exists(
+            repo_id=repo_id,
+            repo_type="dataset",
+        )
+        if exists is not True:
+            self.hf_api.create_repo(
+                repo_id=repo_id,
+                repo_type="dataset",
+                private=False,
+                exist_ok=True
+            )
+        exists: bool = self.hf_api.repo_exists(
+            repo_id=repo_id,
+            repo_type="dataset",
+        )
+        return exists
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--key_of_credentials",
+        default="hf_miyuki_write_token",
+        type=str,
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    import log
+    from project_settings import environment, project_path, log_directory, time_zone_info
+
+    log.setup_size_rotating(log_directory=log_directory, tz_info=time_zone_info)
+
+    hf_token = environment.get(key=args.key_of_credentials)
+
+    client = RepoClient(
+        hf_token=hf_token,
+    )
+
+    flag = client.check_repo("tianxing1994/hf_netdisk")
+    print(f"flag: {flag}")
+    repo_id = client.create_repo("tianxing1994/hf_netdisk")
+    flag = client.check_repo("tianxing1994/hf_netdisk")
+    print(f"flag: {flag}")
+    repo_id = client.delete_repo("tianxing1994/hf_netdisk")
+    flag = client.check_repo("tianxing1994/hf_netdisk")
+    print(f"flag: {flag}")
+
+    return
+
+
+if __name__ == "__main__":
+    main()

toolbox/porter/tasks/__init__.py

@@ -6,6 +6,7 @@ from .douyin_video_download_task import DouyinVideoDownloadTask
 from .douyin_live_record_task import DouyinLiveRecordTask
 from .douyin_live_to_bilibili_live_task import DouyinLiveToBilibiliLiveTask
 from .file_to_baidu_netdisk_task import FileToBaiduNetdiskTask
+from .file_to_hf_netdisk_task import FileToHfNetdiskTask
 from .video_format_convert_task import VideoFormatConvertTask
 from .video_to_bilibili_task import VideoToBilibiliTask
 from .video_to_youtube_task import VideoToYoutubeTask

toolbox/porter/tasks/file_to_hf_netdisk_task.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import aiofiles
+import asyncio
+import copy
+from datetime import datetime
+from zoneinfo import ZoneInfo  # Python 3.9+ 自带，无需安装
+import logging
+import json
+import os
+from pathlib import Path
+from typing import Coroutine, Dict, List, Tuple, Union, Iterable
+
+logger = logging.getLogger("toolbox")
+
+from toolbox.porter.tasks.base_task import BaseTask
+from toolbox.hf_netdisk.netdisk.netdisk_client import NetdiskClient
+from project_settings import environment, project_path, time_zone_info
+
+
+@BaseTask.register("file_to_hf_netdisk")
+class FileToHfNetdiskTask(BaseTask):
+    def __init__(self,
+                 task_name: str,
+                 src_dir: str,
+                 tgt_dir: str,
+                 repo_id: str,
+                 check_interval: int,
+                 key_of_credentials: str,
+                 remove_after_upload: bool = False,
+                 exclude_files: List[str] = None,
+                 **kwargs
+                 ):
+        super().__init__(
+            flag=f"[{self.__class__.__name__}_{task_name}]",
+            check_interval=check_interval
+        )
+        self.task_name = task_name
+        self.tgt_dir: str = tgt_dir
+        # tgt_dir
+        # xianyu/customers/{task_name}/{date_str}/{time_str}
+        self.repo_id: str = repo_id
+
+        self.remove_after_upload = remove_after_upload
+        self.exclude_files = exclude_files or list()
+        self.key_of_credentials = key_of_credentials
+
+        if not os.path.isabs(src_dir):
+            self.src_dir: Path = project_path / src_dir
+        else:
+            self.src_dir: Path = Path(src_dir)
+
+        hf_token = environment.get(self.key_of_credentials, dtype=str)
+        self.client = NetdiskClient(hf_token=hf_token)
+
+    async def do_task(self):
+        for filename in self.src_dir.glob("*"):
+            name = filename.name
+            if name in self.exclude_files:
+                continue
+
+            date_now_str = datetime.now().strftime("%Y%m%d")
+            time_now_str = datetime.now().strftime("%H%M%S")
+
+            kwargs = {
+                "task_name": self.task_name,
+                "date_str": date_now_str,
+                "time_str": time_now_str,
+            }
+
+            tgt_dir = self.tgt_dir.format(**kwargs)
+            tgt_file = Path(tgt_dir) / name
+
+            logger.info(f"{self.flag}上传视频; repo_id: {self.repo_id}, src_file: {filename.as_posix()}, tgt_file: {tgt_file.as_posix()}")
+            # _ = await asyncio.to_thread(
+            #     self.client.upload_file,
+            #     src_file=filename.as_posix(),
+            #     tgt_file=tgt_file.as_posix()
+            # )
+            tgt_file_url: str = self.client.upload_file(
+                src_file=filename.as_posix(),
+                tgt_file=tgt_file.as_posix(),
+                repo_id=self.repo_id,
+            )
+            if self.remove_after_upload:
+                logger.info(f"{self.flag}删除本地文件：{filename.as_posix()}")
+                os.remove(filename.as_posix())
+            logger.info(f"{self.flag}上传视频成功; local_file: {filename.as_posix()}, remote:file: {tgt_file_url}")
+
+
+def main():
+    return
+
+
+if __name__ == "__main__":
+    main()

toolbox/porter/tasks/video_format_convert_task.py

@@ -62,7 +62,6 @@ class VideoFormatConvertTask(BaseTask):
 
     async def do_task(self):
         video_info = await self.load_video_info()
-        print(self.video_info_file)
 
         new_video_info = copy.deepcopy(video_info)
         for k, v in video_info.items():
@@ -104,9 +103,8 @@ class VideoFormatConvertTask(BaseTask):
         output_file = self.target_dir / output_file.name
         output_file.parent.mkdir(parents=True, exist_ok=True)
 
-        idx = uuid.uuid4()
-        input_file_ = input_file.parent / f"{idx}{input_file.suffix}"
-        output_file_ = output_file.parent / f"{idx}{output_file.suffix}"
+        input_file_ = input_file.parent / f"{uuid.uuid4()}{input_file.suffix}"
+        output_file_ = input_file.parent / f"{uuid.uuid4()}{output_file.suffix}"
         os.rename(input_file.as_posix(), input_file_.as_posix())
 
         command = [