1

.dockerignore

@@ -0,0 +1 @@
+.venv/

.gitignore

@@ -0,0 +1,5 @@
+.venv/
+.venv
+
+__pychahe__/
+*.pyc

.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+    
+        {
+            "name": "Python Debugger: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            // 关键改动：${fileDirname} 会让调试器自动进入 run.py 所在的目录（即 jable 目录）
+            "cwd": "${fileDirname}",
+            "env":{
+                "PYTHONPATH":"${workspaceFolder}"
+            },
+            "console": "integratedTerminal"
+        }
+    ]
+}

Dockerfile

@@ -4,11 +4,22 @@ FROM ubuntu:24.04
 # 避免交互式提示（tzdata 等）
 ENV DEBIAN_FRONTEND=noninteractive
 
+# 24.04 路径略有不同
+RUN sed -i 's/archive.ubuntu.com/mirrors.aliyun.com/g' /etc/apt/sources.list.d/ubuntu.sources && \
+    sed -i 's/security.ubuntu.com/mirrors.aliyun.com/g' /etc/apt/sources.list.d/ubuntu.sources
+
+# 定义时区变量
+ENV TZ=Asia/Shanghai
+
 # 1. 安装 sudo（很多基础镜像没有 sudo）
 RUN apt-get update && \
     apt-get install -y sudo unzip lsof python3 python3-pip python3-venv curl git wget nodejs npm xz-utils zip libglu1-mesa ffmpeg && \
     # 清理缓存，减小镜像体积
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+    ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
+    echo $TZ > /etc/timezone && \
+    apt-get install -y tzdata && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # 2. 创建新用户（这里叫 devuser，你可以改名）
 #    -m 创建 home 目录

jable/baidu.py

@@ -0,0 +1,123 @@
+import DrissionPage 
+from time import sleep
+
+from pathlib import Path
+
+import os
+
+def start(ts_path: Path, m3u8_path: Path, download_path: Path):
+
+    pngs = list(ts_path.glob('*.png'))
+
+    # total_count = 20
+    total_count = len(pngs)
+    batch_size = 5
+    options = DrissionPage.ChromiumOptions()
+    options.headless = False
+    options.no_imgs = True
+    options.incognito = True
+
+    page = DrissionPage.Chromium(options)
+    tab = page.latest_tab
+    tab.set.window.max()
+    tab.get("https://www.baidu.com")
+
+    # print(tab.html)
+    # ele_ment = tab.ele('xpath://*[@id="right-tool"]/div[3]/div[2]/div/div')
+    upload_api = 'aichat/api/file/upload'
+    base_cdn = 'https://aisearch.cdn.bcebos.com/'
+    
+    image_cdns = []
+
+        # 2. 开始大循环
+    for i in range(0, total_count, batch_size):
+        print(f"--- 正在执行第 {i + 1} 到 {i + batch_size} 次上传 ---")
+        
+        # 【关键改动】每一轮刷新后，必须重新获取元素对象，否则会报错
+        # 建议先等待元素出现，确保页面加载完成
+        ele_ment = tab.ele('xpath://*[@id="right-tool"]/div[3]/div[2]/div/div', timeout=10)
+        
+        if not ele_ment:
+            print("未找到上传元素，跳过本轮")
+            tab.refresh()
+            sleep(1)
+            continue
+
+        # 3. 准备本轮监听
+        tab.listen.start(upload_api)
+
+        # 4. 准备本轮文件列表并上传
+        # urls = [file_path for _ in range(batch_size)]                                                                         
+        urls: list[str] = []
+        batch_pages = range(i, min(i + batch_size, total_count))
+        for j in batch_pages:
+            url = pngs[j]
+            urls.append(str(url))
+
+        print(f"正在上传 {len(urls)} 个文件...")
+        print(urls)
+        ele_ment.click.to_upload(urls)
+
+        # 5. 等待数据包捕获
+        # count 设置为 batch_size，fit_count=True 表示必须集齐10个响应才继续（除非超时）
+        datas = tab.listen.wait(count=batch_size, timeout=15, fit_count=False)
+        
+        if datas:
+
+            print(f"成功捕获到 {len(datas)} 个响应包")
+            # 这里可以遍历 datas 处理响应结果，例如获取文件 ID
+            for packet in datas:
+                print(packet.request)
+                pd = packet.request.postData
+                name = pd.get('name', '')
+                url = base_cdn + pd.get('path')
+                image_cdns.append((name, url))
+        else:
+            print("警告：本轮未捕获到响应数据，可能上传失败或超时")
+
+        # 6. 刷新页面，清理状态
+        print("本轮结束，正在刷新页面...")
+        tab.refresh()
+        
+        # 7. 刷新后稍微等待，确保页面稳定（或使用 wait.load_start()）
+        sleep(1)
+    
+    print(f"上传任务已完成。下载成功 {len(image_cdns)}")
+
+    # print(image_cdns)
+    sorted_image_cdns = sorted(image_cdns, key=sort_by_name_number)
+    print(sorted_image_cdns)
+
+    lines = []
+    timeLines = []
+    m3name = str(ts_path.parent).split(os.sep)[-1] + '.m3u8'
+    with open(m3u8_path, 'r', encoding='utf-8') as f:
+        lines = f.readlines()
+    to_m3u8Path = download_path.joinpath(m3name)
+
+    for str1 in lines:
+        if str1.startswith('#EXTINF'):
+            timeLines.append(str1)
+    
+    with open(to_m3u8Path, 'w', encoding='utf-8') as f:
+        f.write('#EXTM3U\n')
+        f.write('#EXT-X-VERSION:3\n')
+        f.write('#EXT-X-MEDIA-SEQUENCE:0\n')
+        f.write('#EXT-X-ALLOW-CACHE:YES\n')
+        f.write('#EXT-X-TARGETDURATION:14\n')
+
+        for idx, (name, tmp_url) in enumerate(sorted_image_cdns):
+            f.write(timeLines[idx])
+            f.write(tmp_url+'\n')
+
+        f.write('#EXT-X-ENDLIST')
+
+def sort_by_name_number(item):
+
+    # 步骤1：拆分文件名和扩展名（000.png → ["000", "png"]）
+    name_without_ext = item[0].split('.')[0]
+    # 步骤2：将数字字符串转为整数（"000" → 0，"010" → 10）
+    return int(name_without_ext)
+
+if __name__ == "__main__":
+    pass

jable/jable/items.py

@@ -0,0 +1,12 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class JableItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass

jable/jable/middlewares.py

@@ -0,0 +1,100 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class JableSpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    async def process_start(self, start):
+        # Called with an async iterator over the spider start() method or the
+        # matching method of an earlier spider middleware.
+        async for item_or_request in start:
+            yield item_or_request
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)
+
+
+class JableDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)

jable/jable/pipelines.py

@@ -0,0 +1,13 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class JablePipeline:
+    def process_item(self, item, spider):
+        return item

jable/jable/settings.py

@@ -0,0 +1,89 @@
+# Scrapy settings for jable project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = "jable"
+
+SPIDER_MODULES = ["jable.spiders"]
+NEWSPIDER_MODULE = "jable.spiders"
+
+ADDONS = {}
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+LOG_LEVEL = "ERROR"
+
+# Concurrency and throttling settings
+CONCURRENT_REQUESTS = 16
+CONCURRENT_REQUESTS_PER_DOMAIN = 16
+DOWNLOAD_DELAY = 0
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+#    "Accept-Language": "en",
+#}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    "jable.middlewares.JableSpiderMiddleware": 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    "jable.middlewares.JableDownloaderMiddleware": 543,
+#}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    "scrapy.extensions.telnet.TelnetConsole": None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    "jable.pipelines.JablePipeline": 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = "httpcache"
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+
+# Set settings whose default value is deprecated to a future-proof value
+FEED_EXPORT_ENCODING = "utf-8"

jable/jable/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

jable/jable/spiders/example.py

@@ -0,0 +1,26 @@
+import scrapy
+
+
+from upload import upload
+
+import time
+URLS = []
+class ExampleSpider(scrapy.Spider):
+    name = "example"
+    # allowed_domains = ["example.com"]
+    start_urls = ["https://www.baidu.com"]
+
+    def parse(self, response):
+        URLS.append(('awd-31221', 'https://play.kvmplay.org/m3/680e95e1772260488/50NjlhMWUzYzhkYTAwOSQzNTg4JDEkMTc3MjIxNzI4OA69a1e3c8da006.m3u8'))
+        # URLS.append(('awd-39999', 'https://play.kvmplay.org/m3/45afe831772263170/39NjlhMWVlNDJjNTc2ZCQzNTg4JDIkMTc3MjIxOTk3MA69a1ee42c576b.m3u8'))
+        pass
+
+    def closed(self, reason):
+        print('closed ', reason)
+        
+        start_time = time.perf_counter()
+
+        upload(URLS)
+        end_time = time.perf_counter() - start_time
+        print(f'结束了 --- {end_time}')
+

jable/run.py

@@ -0,0 +1,12 @@
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.project import get_project_settings
+from jable.spiders.example import ExampleSpider
+
+
+
+
+if __name__ == "__main__":
+
+    process = CrawlerProcess(get_project_settings())
+    process.crawl(ExampleSpider)
+    process.start()

jable/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = jable.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = jable

jable/upload.py

@@ -0,0 +1,53 @@
+
+from pathlib import Path
+import os
+from baidu import start as baidu_start
+import subprocess
+def changeTSToPng(ts_path: Path):
+
+    png_path = Path(__file__).parent.joinpath('hide.png')
+    print(png_path)
+    ts_datas = list(ts_path.glob('*.ts'))
+    for ts in ts_datas:
+        # print(ts)
+        ts_name = str(ts).split(os.sep)[-1].split('.')[0] + '.png'
+        with open(ts_path.joinpath(ts_name), 'wb') as f:
+            with open(png_path, 'rb') as png_f:
+                f.write(png_f.read())
+            with open(ts, 'rb') as ts1:
+                f.write(ts1.read())
+        ts.unlink()
+        
+def upload(URLS):
+    tmp_path = Path.home().joinpath('tmps')
+    download_path = Path.home().joinpath('download_videos')
+    
+    print(tmp_path)
+    # return
+    for name, url in URLS:
+        ts_path = tmp_path.joinpath(name).joinpath('0____')
+        m3u8_path = ts_path.parent.joinpath('raw.m3u8')
+        ARGS = [
+            "N_m3u8DL-RE",
+            url,
+            "--thread-count",
+            "11",
+            "--skip-merge",
+            "True",
+            "--tmp-dir",
+            tmp_path,
+            "--save-dir",
+            download_path,
+            "--save-name",
+            name,
+            "-H",
+            "Origin: https://www.4kvm.net",
+            "-H",
+            "user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
+        ]
+        subprocess.run(ARGS)
+
+        # 替换ts 文件变成 png
+        changeTSToPng(ts_path)
+        baidu_start(ts_path, m3u8_path, download_path)
+        print("*" * 50)