Dockerfile

FROM gitea/gitea:1.24.6

# 以 root 用户设置权限（Spaces 允许构建时 root）
USER root

# 安装 Python、pip、git 和异步库
RUN apk update && \
    apk add python3 py3-pip git && \
    pip3 install --break-system-packages --upgrade pip

# 安装 Python 包，包括 pytz 用于处理时区
RUN pip3 install --break-system-packages watchdog huggingface_hub aiohttp pytz

# 预创建 Gitea 所需目录并设置权限
RUN mkdir -p /data/gitea/conf /data/gitea/log /data/gitea/git /data/git /data/ssh && \
    chown -R git:git /data && \
    chmod -R 770 /data

# 复制拉取 Hugging Face 数据集的脚本，硬编码 base64 解码的 HF_TOKEN
COPY --chown=root:root <<'EOF' /pullhf.py
import os
import shutil
from huggingface_hub import snapshot_download
import logging
import base64
from datetime import datetime
import pytz

# 配置日志，使用北京时间
beijing_tz = pytz.timezone('Asia/Shanghai')
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S %Z'
)
logger = logging.getLogger(__name__)

# 自定义日志格式化器以使用北京时间
class BeijingTimeFormatter(logging.Formatter):
    def formatTime(self, record, datefmt=None):
        dt = datetime.fromtimestamp(record.created, tz=beijing_tz)
        if datefmt:
            return dt.strftime(datefmt)
        return dt.strftime("%Y-%m-%d %H:%M:%S %Z")

# 应用自定义格式化器
for handler in logging.getLogger().handlers:
    handler.setFormatter(BeijingTimeFormatter(
        fmt='%(asctime)s - %(levelname)s - %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S %Z'
    ))

def pull_from_hf_hub(repo_id, data_directory="/data"):
    """从 Hugging Face Hub 数据集仓库拉取数据替换 /data 目录"""
    # 硬编码的 base64 编码的 HF_TOKEN
    hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
    hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
    
    try:
        # 调试：打印 HF_TOKEN
        logger.info(f"HF_TOKEN value: {hf_token}")
        # 临时目录用于下载
        temp_dir = "/tmp/hf_download"
        if os.path.exists(temp_dir):
            shutil.rmtree(temp_dir, ignore_errors=True)
        
        # 下载 Hugging Face 数据集
        logger.info(f"正在从 Hugging Face Hub 拉取数据集: {repo_id}")
        snapshot_download(
            repo_id=repo_id,
            repo_type="dataset",
            local_dir=temp_dir,
            token=hf_token,
            ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
        )

        # 清空现有的 /data 目录
        if os.path.exists(data_directory):
            logger.info(f"正在清空现有 /data 目录: {data_directory}")
            shutil.rmtree(data_directory, ignore_errors=True)
        
        # 创建 /data 目录并移动下载的内容
        os.makedirs(data_directory, exist_ok=True)
        for item in os.listdir(temp_dir):
            src = os.path.join(temp_dir, item)
            dst = os.path.join(data_directory, item)
            shutil.move(src, dst)
        
        # 清理临时目录
        shutil.rmtree(temp_dir, ignore_errors=True)
        
        # 检查 /data/gitea 是否存在
        gitea_dir = "/data/gitea"
        if os.path.exists(gitea_dir):
            logger.info(f"/data/gitea 存在，将跳过 Gitea 初始化页面")
            # 设置环境变量文件以跳过初始化
            with open("/data/gitea_skip_install", "w") as f:
                f.write("true")
        else:
            logger.info(f"/data/gitea 不存在，允许 Gitea 进入初始化页面")
            # 创建 Gitea 必需目录
            for dir_path in ["/data/gitea/conf", "/data/gitea/log", "/data/gitea/git", "/data/git", "/data/ssh"]:
                os.makedirs(dir_path, exist_ok=True)
        
        # 修复权限：确保 /data 及其子目录为 git:git 所有且可写
        logger.info("修复 /data 目录及其子目录的权限")
        os.system("chown -R git:git /data")
        os.system("chmod -R 770 /data")
        
        # 调试：列出 /data 目录结构和权限
        logger.info("调试：列出 /data 目录结构和权限")
        os.system("ls -la /data")
        os.system("ls -la /data/gitea 2>/dev/null || echo '/data/gitea 不存在'")
        os.system("ls -la /data/gitea/conf 2>/dev/null || echo '/data/gitea/conf 不存在'")
        os.system("ls -la /data/gitea/log 2>/dev/null || echo '/data/gitea/log 不存在'")
        
        logger.info(f"成功从 Hugging Face Hub 拉取数据到 {data_directory}")
        return True

    except Exception as e:
        logger.error(f"拉取 Hugging Face 数据集失败: {e}")
        return False

if __name__ == "__main__":
    repo_id = os.getenv("REPO_ID", "02engine/02gitea")
    if not pull_from_hf_hub(repo_id):
        logger.error("拉取数据集失败，退出")
        exit(1)
EOF

COPY --chown=git:git <<'EOF' /uploadhf.py
import os
import time
import logging
import threading
import subprocess
import asyncio
import aiohttp
from pathlib import Path
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from huggingface_hub import HfApi
# 删除：from huggingface_hub.utils import tqdm_utils
import base64
from datetime import datetime
import pytz

# 配置日志，使用北京时间
beijing_tz = pytz.timezone('Asia/Shanghai')
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S %Z'
)
logger = logging.getLogger(__name__)

# 自定义日志格式化器以使用北京时间
class BeijingTimeFormatter(logging.Formatter):
    def formatTime(self, record, datefmt=None):
        dt = datetime.fromtimestamp(record.created, tz=beijing_tz)
        if datefmt:
            return dt.strftime(datefmt)
        return dt.strftime("%Y-%m-%d %H:%M:%S %Z")

# 应用自定义格式化器
for handler in logging.getLogger().handlers:
    handler.setFormatter(BeijingTimeFormatter(
        fmt='%(asctime)s - %(levelname)s - %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S %Z'
    ))

class DataDirectoryHandler(FileSystemEventHandler):
    """处理 /data 目录文件变化的监控器"""
    
    def __init__(self, repo_id, hf_token, data_directory="/data"):
        self.repo_id = repo_id
        self.hf_token = hf_token
        self.data_directory = data_directory
        self.api = HfApi(token=hf_token)
        self.last_commit_time = 0
        self.commit_delay = 1
        self.pending_changes = []
        logger.info(f"初始化监控器，监控目录: {data_directory}，目标仓库: {repo_id}")
    
    def on_any_event(self, event):
        if event.is_directory:
            return
        self.pending_changes.append((event.event_type, event.src_path))
        self.schedule_commit(f"文件{event.event_type}")

    def schedule_commit(self, change_type):
        current_time = time.time()
        if current_time - self.last_commit_time > self.commit_delay:
            self.last_commit_time = current_time
            asyncio.run(self.commit_changes(change_type))

    async def commit_changes(self, change_type):
        max_retries = 3
        retry_delay = 5
        change_summary = f"{change_type} ({len(self.pending_changes)} 文件)"
        self.pending_changes = []
        for attempt in range(max_retries):
            try:
                commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
                logger.info(f"开始上传: {commit_message}")
                await asyncio.to_thread(
                    self.api.upload_folder,
                    folder_path=self.data_directory,
                    repo_id=self.repo_id,
                    repo_type="dataset",
                    commit_message=commit_message,
                    ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"],
                    disable_progress_bar=True  # 正确禁用进度条
                )
                logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
                return
            except Exception as e:
                logger.error(f"提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
                if attempt < max_retries - 1:
                    logger.info(f"将在 {retry_delay} 秒后重试...")
                    await asyncio.sleep(retry_delay)
        logger.error(f"达到最大重试次数，上传失败")

def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
    if not hf_token:
        hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
        hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
        logger.info(f"HF_TOKEN value: {hf_token}")
    
    if not repo_id:
        raise ValueError("必须提供 repo_id 参数")
    
    if not os.path.exists(data_directory):
        logger.warning(f"监控目录 {data_directory} 不存在，正在创建...")
        os.makedirs(data_directory, exist_ok=True)
    
    event_handler = DataDirectoryHandler(
        repo_id=repo_id,
        hf_token=hf_token,
        data_directory=data_directory
    )
    
    observer = Observer()
    observer.schedule(event_handler, data_directory, recursive=True)
    observer.start()
    logger.info(f"目录监控服务已启动: {data_directory}")
    
    return observer

def start_gitea_server(port=7860):
    def run_gitea():
        try:
            gitea_process = subprocess.Popen(
                ['gitea', 'web', '--port', str(port)],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                universal_newlines=True
            )
            logger.info(f"Gitea 服务器已启动，端口: {port}")
            logger.info(f"访问地址: http://localhost:{port}")
            
            while True:
                output = gitea_process.stdout.readline()
                if output == '' and gitea_process.poll() is not None:
                    break
                if output:
                    logger.info(f"Gitea: {output.strip()}")
            
            return_code = gitea_process.poll()
            if return_code != 0:
                error_output = gitea_process.stderr.read()
                logger.error(f"Gitea 服务器异常退出，返回码: {return_code}")
                logger.error(f"错误信息: {error_output}")
            else:
                logger.info("Gitea 服务器正常退出")
                
        except FileNotFoundError:
            logger.error("未找到 gitea 命令，请确保 Gitea 已正确安装")
        except Exception as e:
            logger.error(f"启动 Gitea 服务器时发生错误: {e}")
    
    gitea_thread = threading.Thread(target=run_gitea)
    gitea_thread.daemon = True
    gitea_thread.start()
    
    return gitea_thread

async def main():
    CONFIG = {
        "data_directory": "/data",
        "repo_id": os.getenv("REPO_ID", "02engine/02gitea"),
        "hf_token": os.getenv('HF_TOKEN'),
        "gitea_port": 7860
    }
    
    max_retries = 5
    retry_count = 0
    
    while retry_count < max_retries:
        try:
            logger.info(f"启动集成服务 (尝试 {retry_count + 1}/{max_retries})...")
            
            logger.info("运行 pullhf.py 拉取最新数据集")
            pull_result = subprocess.run(["python3", "/pullhf.py"], check=True)
            if pull_result.returncode != 0:
                logger.error("pullhf.py 执行失败，退出")
                exit(1)
            
            gitea_thread = start_gitea_server(CONFIG["gitea_port"])
            observer = start_directory_monitoring(
                data_directory=CONFIG["data_directory"],
                repo_id=CONFIG["repo_id"],
                hf_token=CONFIG["hf_token"]
            )
            
            logger.info("所有服务已启动完成！")
            logger.info("目录监控: /data to Hugging Face Hub")
            logger.info("Gitea 服务: http://localhost:7860")
            logger.info("按 Ctrl+C 停止所有服务")
            
            try:
                while True:
                    await asyncio.sleep(1)
            except KeyboardInterrupt:
                logger.info("正在停止服务...")
                break
            
        except Exception as e:
            retry_count += 1
            logger.error(f"服务崩溃 (尝试 {retry_count}/{max_retries}): {e}")
            if retry_count < max_retries:
                logger.info(f"将在 5 秒后尝试重启...")
                await asyncio.sleep(5)
            else:
                logger.error(f"达到最大重试次数 ({max_retries})，退出")
                exit(1)
        finally:
            if 'observer' in locals():
                observer.stop()
                observer.join()
            logger.info("所有服务已停止")

if __name__ == "__main__":
    asyncio.run(main())
EOF

# 设置工作目录
WORKDIR /data

# 暴露 Spaces 默认端口（文档用途，实际由 $PORT 控制）
EXPOSE 7860

# Gitea 环境变量：适配 Spaces 端口，禁用 SSH，动态决定是否跳过安装页面
RUN echo '#!/bin/sh' > /set_gitea_env.sh && \
    echo 'if [ -f /data/gitea_skip_install ]; then' >> /set_gitea_env.sh && \
    echo '  export GITEA__server__STARTUP_DISABLE_INSTALL_FORM=true' >> /set_gitea_env.sh && \
    echo 'else' >> /set_gitea_env.sh && \
    echo '  export GITEA__server__STARTUP_DISABLE_INSTALL_FORM=false' >> /set_gitea_env.sh && \
    echo 'fi' >> /set_gitea_env.sh && \
    echo 'exec "$@"' >> /set_gitea_env.sh && \
    chmod +x /set_gitea_env.sh

ENV GITEA__database__DB_TYPE=sqlite3 \
    GITEA__database__PATH=/data/gitea/gitea.db \
    GITEA__server__ROOT_URL=https://deep-sea-02gitea.hf.space/ \
    GITEA__server__HTTP_ADDR=0.0.0.0 \
    GITEA__server__HTTP_PORT=7860 \
    GITEA__server__DISABLE_SSH=true \
    USER=git

# 切换到 git 用户（非 root）
USER git

# 运行 Gitea，显式指定 Spaces 默认端口
CMD ["/set_gitea_env.sh", "python3", "/uploadhf.py"]