update

data/porter_tasks/porter_task_chenjieshen_douyin_video_to_bilibili.json

@@ -6,7 +6,7 @@
     "sec_user_id": "MS4wLjABAAAATGoBrO7yiJ3q9go4fxq9JXjrnP1bFpdkgKckC1IpfXA_vrjSmL9ZtjmTju8ApwbT",
     "check_interval": 900,
     "key_of_credentials": "douyin_wentao_credentials",
-    "min_date2": "2025-09-09 00:00:00",
+    "min_date2": "2025-09-12 00:00:00",
     "output_video_dir": "data/tasks/chenjieshen_douyin_video_to_bilibili/video/douyin/陈杰森",
     "output_video_info_file": "data/tasks/chenjieshen_douyin_video_to_bilibili/video/douyin/陈杰森/file_info.json"
   },
@@ -17,7 +17,7 @@
     "sec_user_id": "MS4wLjABAAAA49QFP6YhorLIIX9M-FiZeKxmqhqXlttluSsZeaxvxzU",
     "check_interval": 900,
     "key_of_credentials": "douyin_wentao_credentials",
-    "min_date2": "2025-09-09 00:00:00",
+    "min_date2": "2025-09-12 00:00:00",
     "output_video_dir": "data/tasks/chenjieshen_douyin_video_to_bilibili/video/douyin/陈杰森",
     "output_video_info_file": "data/tasks/chenjieshen_douyin_video_to_bilibili/video/douyin/陈杰森/file_info.json"
   },
@@ -31,6 +31,6 @@
     "target_user_id": "442286660",
     "key_of_credentials": "bilibili_chenjiesen_credentials",
     "remove_after_upload": true,
-    "min_date": "2025-09-09 00:00:00"
+    "min_date2": "2025-09-12 00:00:00"
   }
 ]

toolbox/bilibili/video/video_manager.py

@@ -5,6 +5,7 @@ https://pypi.org/project/biliupload/
 https://github.com/SocialSisterYi/bilibili-API-collect
 """
 import argparse
+import json
 import logging
 import math
 from pathlib import Path
@@ -12,6 +13,7 @@ import re
 
 logger = logging.getLogger("toolbox")
 
+import requests
 from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 from tqdm import tqdm
 
@@ -28,7 +30,7 @@ class BilibiliVideoUploader20200810(BilibiliClient):
         stop=stop_after_attempt(3),
         before_sleep=before_sleep_log(logger, logging.ERROR),
     )
-    def preupload(self, filename, filesize):
+    def pre_upload(self, filename, filesize):
         url = "https://member.bilibili.com/preupload"
         params = {
             "name":	filename,
@@ -42,12 +44,17 @@ class BilibiliVideoUploader20200810(BilibiliClient):
             "upcdn": "bda2",
             "probe_version": "20200810"
         }
-        response = self.session.get(
-            url,
+
+        response = requests.request(
+            method="POST",
+            url=url,
+            headers={
+                "TE": "Trailers",
+                **self.headers,
+            },
             params=params,
-            headers={"TE": "Trailers"}
+            cookies=self.cookies,
         )
-
         if response.status_code != 200:
             raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
         js = response.json()
@@ -60,11 +67,16 @@ class BilibiliVideoUploader20200810(BilibiliClient):
         stop=stop_after_attempt(3),
         before_sleep=before_sleep_log(logger, logging.ERROR),
     )
-    def get_upload_video_id(self, *, upos_uri, auth):
+    def get_upload_video_id(self, upos_uri, auth, **kwargs):
         url = f"https://upos-sz-upcdnbda2.bilivideo.com/{upos_uri}?uploads&output=json"
-        response = self.session.post(
-            url,
-            headers={"X-Upos-Auth": auth}
+        response = requests.request(
+            method="POST",
+            url=url,
+            headers={
+                "X-Upos-Auth": auth,
+                **self.headers,
+            },
+            cookies=self.cookies,
         )
 
         if response.status_code != 200:
@@ -74,11 +86,11 @@ class BilibiliVideoUploader20200810(BilibiliClient):
             raise AssertionError(f"request failed;")
         return js
 
-    def upload_video_in_chunks(self, *, upos_uri, auth, upload_id, fileio, filesize, chunk_size, chunks):
+    def upload_video_in_chunks(self, endpoint, upos_uri, auth, upload_id, fileio, filesize, chunk_size, chunks):
         """
         Upload the video in chunks.
-        :param upos_uri: str, get from `preupload`
-        :param auth: str, get from `preupload`
+        :param upos_uri: str, get from `pre_upload`
+        :param auth: str, get from `pre_upload`
         :param upload_id: str, get from `get_upload_video_id`
         :param fileio: io.BufferedReader, the io stream of the video to be uploaded
         :param filesize: int, the size of the video to be uploaded
@@ -107,11 +119,16 @@ class BilibiliVideoUploader20200810(BilibiliClient):
             params["size"] = len(batchbytes)
             params["start"] = start
             params["end"] = fileio.tell()
-            response = self.session.put(
-                url,
+            response = requests.request(
+                method="PUT",
+                url=url,
+                headers={
+                    "X-Upos-Auth": auth,
+                    **self.headers,
+                },
                 params=params,
                 data=batchbytes,
-                headers={"X-Upos-Auth": auth}
+                cookies=self.cookies,
             )
             if response.status_code != 200:
                 raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
@@ -122,17 +139,7 @@ class BilibiliVideoUploader20200810(BilibiliClient):
         stop=stop_after_attempt(3),
         before_sleep=before_sleep_log(logger, logging.ERROR),
     )
-    def finish_upload(self, *, upos_uri, auth, filename, upload_id, biz_id, chunks):
-        """
-        Notify the all chunks have been uploaded.
-        :param upos_uri: str, get from `preupload`
-        :param auth: str, get from `preupload`
-        :param filename: str, the name of the video to be uploaded
-        :param upload_id: str, get from `get_upload_video_id`
-        :param biz_id: int, get from `preupload`
-        :param chunks: int, the number of chunks to be uploaded
-        :return:
-        """
+    def finish_upload(self, upos_uri, auth, filename, upload_id, biz_id, chunks):
         url = f"https://upos-sz-upcdnbda2.bilivideo.com/{upos_uri}"
         params = {
             "output": "json",
@@ -147,11 +154,16 @@ class BilibiliVideoUploader20200810(BilibiliClient):
                 for i in range(chunks, 1)
             ]
         }
-        response = self.session.post(
-            url,
+        response = requests.request(
+            method="POST",
+            url=url,
+            headers={
+                "X-Upos-Auth": auth,
+                **self.headers,
+            },
             params=params,
             json=data,
-            headers={"X-Upos-Auth": auth}
+            cookies=self.cookies,
         )
         if response.status_code != 200:
             raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
@@ -180,13 +192,13 @@ class BilibiliVideoUploader20200810(BilibiliClient):
             "videos": [
                 {
                     "filename": bilibili_filename,
-                    "title": metadata["title"],
+                    "title": metadata["title"][:80],
                     "desc": metadata["desc"]
                 }
             ],
             "source": metadata["source"],
             "tid": metadata["tid"],
-            "title": metadata["title"],
+            "title": metadata["title"][:80],
             "cover": metadata["cover"],
             "tag": metadata["tag"],
             "desc_format_id": 0,
@@ -201,10 +213,15 @@ class BilibiliVideoUploader20200810(BilibiliClient):
             # copyright: 1 original 2 reprint
             data["copyright"] = 1
 
-        response = self.session.post(
-            url,
+        response = requests.request(
+            method="POST",
+            url=url,
+            headers={
+                "TE": "Trailers",
+                **self.headers,
+            },
             json=data,
-            headers={"TE": "Trailers"}
+            cookies=self.cookies,
         )
         if response.status_code != 200:
             raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
@@ -215,7 +232,7 @@ class BilibiliVideoUploader20200810(BilibiliClient):
         filename = Path(filename)
         filesize = filename.stat().st_size
 
-        js = self.preupload(filename=filename, filesize=filesize)
+        js = self.pre_upload(filename=filename, filesize=filesize)
         upos_uri = js["upos_uri"].split("//")[-1]
         auth = js["auth"]
         biz_id = js["biz_id"]
@@ -229,6 +246,7 @@ class BilibiliVideoUploader20200810(BilibiliClient):
         bilibili_filename = re.search(r"/(.*)\.", key).group(1)
         fileio = filename.open(mode="rb")
         self.upload_video_in_chunks(
+            endpoint="",
             upos_uri=upos_uri,
             auth=auth,
             upload_id=upload_id,
@@ -253,32 +271,349 @@ class BilibiliVideoUploader20200810(BilibiliClient):
 
         bvid = None
         status_code = publish_video_response["code"]
-        if status_code == 137022:
+        if status_code != 0:
+            # 137022
+            # 21104, 第(1)个视频的标题过长,已经超过80个字符
             message = publish_video_response["message"]
             logger.info(f"publish_video failed; code: {status_code}, message: {message}")
         else:
-            bvid = publish_video_response["data"]["bvid"]
+            try:
+                bvid = publish_video_response["data"]["bvid"]
+            except KeyError as error:
+                raise KeyError(f"publish_video failed; KeyError: {error}, publish_video_response: {publish_video_response}")
         return bvid
 
+    @retry(
+        wait=wait_fixed(10),
+        stop=stop_after_attempt(3),
+        before_sleep=before_sleep_log(logger, logging.ERROR),
+    )
+    def predict_tid(self, title: str, upload_id: str, bilibili_filename: str):
+        csrf = self.cookies["bili_jct"]
+        url = f"https://member.bilibili.com/x/vupre/web/archive/types/predict"
+        params = {
+            "csrf": csrf,
+        }
+        data = {
+            "title": title,
+            "upload_id": upload_id,
+            "filename": bilibili_filename,
+        }
+        response = requests.request(
+            method="POST",
+            url=url,
+            headers={
+                "TE": "Trailers",
+                **self.headers,
+            },
+            params=params,
+            json=data,
+            cookies=self.cookies,
+        )
+        if response.status_code != 200:
+            raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
+        js = response.json()
+        return js
+
 
 class BilibiliVideoUploader20221109(BilibiliVideoUploader20200810):
     def __init__(self):
         super().__init__()
-        
+
+    @retry(
+        wait=wait_fixed(10),
+        stop=stop_after_attempt(3),
+        before_sleep=before_sleep_log(logger, logging.ERROR),
+    )
+    def pre_upload(self, filename, filesize):
+        url = "https://member.bilibili.com/preupload"
+        params = {
+            "probe_version": "20221109",
+            "upcdn": "bldsa",
+            "zone": "cs",
+            "name": filename,
+            "r": "upos",
+            "profile": "ugcfx/bup",
+            "ssl": 0,
+            "version": "2.14.0.0",
+            "build": "2140000",
+            "size": filesize,
+        }
+        # response = self.session.get(
+        #     url,
+        #     params=params,
+        #     headers={"TE": "Trailers"}
+        # )
+        response = requests.request(
+            # method="GET",
+            method="POST",
+            url=url,
+            headers={
+                "TE": "Trailers",
+                **self.headers,
+            },
+            params=params,
+            cookies=self.cookies,
+        )
+
+        if response.status_code != 200:
+            raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
+        js = response.json()
+        if js["OK"] != 1:
+            raise AssertionError(f"request failed;")
+        return js
+
+    @retry(
+        wait=wait_fixed(10),
+        stop=stop_after_attempt(3),
+        before_sleep=before_sleep_log(logger, logging.ERROR),
+    )
+    def get_upload_video_id(self, endpoint, upos_uri, auth,
+                            filesize, chunk_size, biz_id,
+                            **kwargs
+                            ):
+        url = f"https:{endpoint}/{upos_uri}?uploads&output=json"
+        params = {
+            "profile": "ugcfx/bup",
+            "filesize": filesize,
+            "partsize": chunk_size,
+            # "biz_id": biz_id,
+        }
+        response = requests.request(
+            method="POST",
+            url=url,
+            headers={
+                "X-Upos-Auth": auth,
+                **self.headers,
+            },
+            params=params,
+            cookies=self.cookies,
+        )
+
+        if response.status_code != 200:
+            raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
+        js = response.json()
+        if js["OK"] != 1:
+            raise AssertionError(f"request failed;")
+        return js
+
+    def upload_video_in_chunks(self, endpoint, upos_uri, auth, upload_id, fileio, filesize, chunk_size, chunks):
+        url = f"https:{endpoint}/{upos_uri}"
+
+        params = {
+            "partNumber": None,  # start from 1
+            "uploadId":	upload_id,
+            "chunk": None,  # start from 0
+            "chunks": chunks,
+            "size":	None,  # current batch size
+            "start": None,
+            "end":	None,
+            "total": filesize,
+        }
+        process_bar = tqdm(desc="bilibili_upload_video", total=chunks)
+        for chunknum in range(chunks):
+            start = fileio.tell()
+            batchbytes = fileio.read(chunk_size)
+            params["partNumber"] = chunknum + 1
+            params["chunk"] = chunknum
+            params["size"] = len(batchbytes)
+            params["start"] = start
+            params["end"] = fileio.tell()
+            response = requests.request(
+                method="OPTIONS",
+                url=url,
+                headers={
+                    "X-Upos-Auth": auth,
+                    **self.headers,
+                },
+                params=params,
+                data=batchbytes,
+                cookies=self.cookies,
+            )
+            if response.status_code != 200:
+                raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
+            process_bar.update(1)
+
+    @retry(
+        wait=wait_fixed(10),
+        stop=stop_after_attempt(3),
+        before_sleep=before_sleep_log(logger, logging.ERROR),
+    )
+    def finish_upload(self, endpoint, upos_uri, auth, filename, upload_id, biz_id, chunks):
+        url = f"https:{endpoint}/{upos_uri}"
+
+        params = {
+            "output": "json",
+            "name":	filename,
+            "profile" : "ugcfx/bup",
+            "uploadId": upload_id,
+            "biz_id": biz_id
+        }
+        print(params)
+        data = {
+            "parts": [
+                {"partNumber": i+1, "eTag": "etag"}
+                for i in range(chunks)
+            ]
+        }
+        print(data)
+        response = requests.request(
+            method="POST",
+            url=url,
+            headers={
+                "X-Upos-Auth": auth,
+                **self.headers,
+            },
+            params=params,
+            data=json.dumps(data),
+            cookies=self.cookies,
+        )
+        if response.status_code != 200:
+            raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
+        js = response.json()
+        if js["OK"] != 1:
+            raise AssertionError(f"request failed;")
+        return js
+
+    @retry(
+        wait=wait_fixed(10),
+        stop=stop_after_attempt(3),
+        before_sleep=before_sleep_log(logger, logging.ERROR),
+    )
+    def publish_video(self, bilibili_filename, metadata: dict):
+        """
+        publish the uploaded video
+
+        https://github.com/SocialSisterYi/bilibili-API-collect/blob/master/docs/creativecenter/upload.md
+        :param bilibili_filename:
+        :param metadata:
+        :return:
+        """
+        url = f'https://member.bilibili.com/x/vu/web/add?csrf={self.cookies["bili_jct"]}'
+
+        data = {
+            "copyright": metadata["copyright"],
+            "videos": [
+                {
+                    "filename": bilibili_filename,
+                    "title": metadata["title"],
+                    "desc": metadata["desc"]
+                }
+            ],
+            "source": metadata["source"],
+            "tid": metadata["tid"],
+            "title": metadata["title"],
+            "cover": metadata["cover"],
+            "tag": metadata["tag"],
+            "desc_format_id": 0,
+            "desc": metadata["desc"],
+            "dynamic": metadata["dynamic"],
+            "subtitle": {"open": 0, "lan": ""},
+
+        }
+
+        if metadata["copyright"] != 2:
+            del data["source"]
+            # copyright: 1 original 2 reprint
+            data["copyright"] = 1
+
+        response = requests.request(
+            method="POST",
+            url=url,
+            headers={
+                "TE": "Trailers",
+                **self.headers,
+            },
+            json=data,
+            cookies=self.cookies,
+        )
+        if response.status_code != 200:
+            raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
+        js = response.json()
+        return js
+
+    def upload_video_file(self, filename: str):
+        filename = Path(filename)
+        filesize = filename.stat().st_size
+
+        js = self.pre_upload(filename=filename, filesize=filesize)
+        endpoint = js["endpoint"]
+        upos_uri = js["upos_uri"].split("//")[-1]
+        auth = js["auth"]
+        biz_id = js["biz_id"]
+        chunk_size = js["chunk_size"]
+        chunks = math.ceil(filesize / chunk_size)
+
+        upload_video_id_response = self.get_upload_video_id(
+            endpoint=endpoint,
+            upos_uri=upos_uri,
+            auth=auth,
+            filesize=filesize,
+            chunk_size=chunk_size,
+            biz_id=biz_id,
+        )
+        upload_id = upload_video_id_response["upload_id"]
+        key = upload_video_id_response["key"]
+
+        bilibili_filename = re.search(r"/(.*)\.", key).group(1)
+        fileio = filename.open(mode="rb")
+        self.upload_video_in_chunks(
+            endpoint=endpoint,
+            upos_uri=upos_uri,
+            auth=auth,
+            upload_id=upload_id,
+            fileio=fileio,
+            filesize=filesize,
+            chunk_size=chunk_size,
+            chunks=chunks
+        )
+        fileio.close()
+
+        # notify the all chunks have been uploaded
+        self.finish_upload(
+            endpoint=endpoint,
+            upos_uri=upos_uri, auth=auth, filename=filename.name,
+            upload_id=upload_id, biz_id=biz_id, chunks=chunks
+        )
+
+        return biz_id, upload_id, bilibili_filename
+
+    def upload_video_and_publish(self, filename: str, metadata: dict):
+        _, upload_id, bilibili_filename = self.upload_video_file(filename)
+        logger.info(f"finish uploading, upload_id: {upload_id}, bilibili_filename: {bilibili_filename}, filename: {filename}")
+        tid = self.predict_tid(
+            title=metadata["title"],
+            upload_id=upload_id,
+            bilibili_filename=bilibili_filename,
+        )
+        metadata["tid"] = tid
+        logger.info(f"predict tid, tid: {tid}, filename: {filename}")
+        publish_video_response = self.publish_video(bilibili_filename=bilibili_filename, metadata=metadata)
+
+        bvid = None
+        status_code = publish_video_response["code"]
+        if status_code == 137022:
+            message = publish_video_response["message"]
+            logger.info(f"publish_video failed; code: {status_code}, message: {message}")
+        else:
+            bvid = publish_video_response["data"]["bvid"]
+        return bvid
+
 
 BilibiliVideoUploader = BilibiliVideoUploader20200810
+# BilibiliVideoUploader = BilibiliVideoUploader20221109
 
 
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--credentials_file",
-        default=(project_path / "dotenv/bilibili_chenjiesen_credentials.json").as_posix(),
+        "--key_of_credentials",
+        default="bilibili_chenjiesen_credentials",
         type=str
     )
     parser.add_argument(
         "--filename",
-        default=(project_path / "data/video/douyin/陈杰森/[7546905431348612362][20250906_172733]资本市场重大事件 。某行市值第一的隐秘故事。#易会满.mp4").as_posix(),
+        default=(project_path / "data/tasks/chenjieshen_douyin_video_to_bilibili/video/douyin/陈杰森/7548332163859533083_20250910T170000_千万不要帮朋友担保！否则你可能背上一身债！.mp4").as_posix(),
         type=str
     )
     args = parser.parse_args()
@@ -287,7 +622,7 @@ def get_args():
 
 def main():
     import log
-    from project_settings import project_path, log_directory
+    from project_settings import project_path, log_directory, environment
 
     log.setup_size_rotating(log_directory=log_directory)
 
@@ -297,7 +632,8 @@ def main():
 
     flag = client.check_login()
     print(f"flag: {flag}")
-    client.login_with_credentials_file(args.credentials_file)
+    credentials = environment.get(args.key_of_credentials, dtype=json.loads)
+    client.login_with_credentials_info(credentials)
     flag = client.check_login()
     print(f"flag: {flag}")
 

toolbox/bilibili/video/video_manager2.py

@@ -373,32 +373,6 @@ class BilibiliVideoUploader(BilibiliClient):
             bvid = publish_video_response["data"]["bvid"]
         return bvid
 
-    @retry(
-        wait=wait_fixed(10),
-        stop=stop_after_attempt(3),
-        before_sleep=before_sleep_log(logger, logging.ERROR),
-    )
-    def predict_tid(self, title: str, upload_id: str, bilibili_filename: str):
-        csrf = self.cookies["bili_jct"]
-        url = f"https://member.bilibili.com/x/vupre/web/archive/types/predict"
-        params = {
-            "csrf": csrf,
-        }
-        data = {
-            "title": title,
-            "upload_id": upload_id,
-            "filename": bilibili_filename,
-        }
-        response = self.session.post(
-            url,
-            params=params,
-            json=data,
-            headers=self.headers,
-        )
-        if response.status_code != 200:
-            raise AssertionError(f"request failed; status_code: {response.status_code}, text: {response.text}")
-        js = response.json()
-        return js
 
 
 def get_args():