Create Dockerfile

Dockerfile

@@ -0,0 +1,336 @@
+FROM gitea/gitea:1.24.6
+
+# 以 root 用户设置权限（Spaces 允许构建时 root）
+USER root
+
+# 安装 Python、pip、git 和异步库
+RUN apk update && \
+    apk add python3 py3-pip git && \
+    pip3 install --break-system-packages --upgrade pip
+
+# 安装 Python 包
+RUN pip3 install --break-system-packages watchdog huggingface_hub aiohttp
+
+# 预创建 Gitea 所需目录并设置权限
+RUN mkdir -p /data/gitea/conf /data/gitea/log /data/gitea/git /data/git /data/ssh && \
+    chown -R git:git /data && \
+    chmod -R 770 /data
+
+# 复制拉取 Hugging Face 数据集的脚本，硬编码 base64 解码的 HF_TOKEN
+COPY --chown=root:root <<'EOF' /pullhf.py
+import os
+import shutil
+from huggingface_hub import snapshot_download
+import logging
+import base64
+
+# 配置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+def pull_from_hf_hub(repo_id, data_directory="/data"):
+    """从 Hugging Face Hub 数据集仓库拉取数据替换 /data 目录"""
+    # 硬编码的 base64 编码的 HF_TOKEN
+    hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
+    hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
+    
+    try:
+        # 调试：打印 HF_TOKEN
+        logger.info(f"HF_TOKEN value: {hf_token}")
+        # 临时目录用于下载
+        temp_dir = "/tmp/hf_download"
+        if os.path.exists(temp_dir):
+            shutil.rmtree(temp_dir, ignore_errors=True)
+        
+        # 下载 Hugging Face 数据集
+        logger.info(f"正在从 Hugging Face Hub 拉取数据集: {repo_id}")
+        snapshot_download(
+            repo_id=repo_id,
+            repo_type="dataset",
+            local_dir=temp_dir,
+            token=hf_token,
+            ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
+        )
+
+        # 清空现有的 /data 目录
+        if os.path.exists(data_directory):
+            logger.info(f"正在清空现有 /data 目录: {data_directory}")
+            shutil.rmtree(data_directory, ignore_errors=True)
+        
+        # 创建 /data 目录并移动下载的内容
+        os.makedirs(data_directory, exist_ok=True)
+        for item in os.listdir(temp_dir):
+            src = os.path.join(temp_dir, item)
+            dst = os.path.join(data_directory, item)
+            shutil.move(src, dst)
+        
+        # 清理临时目录
+        shutil.rmtree(temp_dir, ignore_errors=True)
+        
+        # 检查 /data/gitea 是否存在
+        gitea_dir = "/data/gitea"
+        if os.path.exists(gitea_dir):
+            logger.info(f"/data/gitea 存在，将跳过 Gitea 初始化页面")
+            # 设置环境变量文件以跳过初始化
+            with open("/data/gitea_skip_install", "w") as f:
+                f.write("true")
+        else:
+            logger.info(f"/data/gitea 不存在，允许 Gitea 进入初始化页面")
+            # 创建 Gitea 必需目录
+            for dir_path in ["/data/gitea/conf", "/data/gitea/log", "/data/gitea/git", "/data/git", "/data/ssh"]:
+                os.makedirs(dir_path, exist_ok=True)
+        
+        # 修复权限：确保 /data 及其子目录为 git:git 所有且可写
+        logger.info("修复 /data 目录及其子目录的权限")
+        os.system("chown -R git:git /data")
+        os.system("chmod -R 770 /data")
+        
+        # 调试：列出 /data 目录结构和权限
+        logger.info("调试：列出 /data 目录结构和权限")
+        os.system("ls -la /data")
+        os.system("ls -la /data/gitea 2>/dev/null || echo '/data/gitea 不存在'")
+        os.system("ls -la /data/gitea/conf 2>/dev/null || echo '/data/gitea/conf 不存在'")
+        os.system("ls -la /data/gitea/log 2>/dev/null || echo '/data/gitea/log 不存在'")
+        
+        logger.info(f"✅ 成功从 Hugging Face Hub 拉取数据到 {data_directory}")
+        return True
+
+    except Exception as e:
+        logger.error(f"❌ 拉取 Hugging Face 数据集失败: {e}")
+        return False
+
+if __name__ == "__main__":
+    repo_id = os.getenv("REPO_ID", "02engine/02gitea")
+    if not pull_from_hf_hub(repo_id):
+        logger.error("❌ 拉取数据集失败，退出")
+        exit(1)
+EOF
+
+# 复制上传脚本，移除检测到事件日志
+COPY --chown=git:git <<'EOF' /uploadhf.py
+import os
+import time
+import logging
+import threading
+import subprocess
+import asyncio
+import aiohttp
+from pathlib import Path
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+from huggingface_hub import HfApi
+import base64
+
+# 配置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+class DataDirectoryHandler(FileSystemEventHandler):
+    """处理 /data 目录文件变化的监控器"""
+    
+    def __init__(self, repo_id, hf_token, data_directory="/data"):
+        self.repo_id = repo_id
+        self.hf_token = hf_token
+        self.data_directory = data_directory
+        self.api = HfApi(token=hf_token)
+        self.last_commit_time = 0
+        self.commit_delay = 1  # 防抖延迟 1 秒
+        self.pending_changes = []  # 缓冲待上传变更
+        logger.info(f"初始化监控器，监控目录: {data_directory}，目标仓库: {repo_id}")
+    
+    def on_any_event(self, event):
+        """捕获所有文件系统事件"""
+        if event.is_directory:
+            return
+        self.pending_changes.append((event.event_type, event.src_path))
+        self.schedule_commit(f"文件{event.event_type}")
+
+    def schedule_commit(self, change_type):
+        """安排提交任务，带有防抖机制"""
+        current_time = time.time()
+        if current_time - self.last_commit_time > self.commit_delay:
+            self.last_commit_time = current_time
+            asyncio.run(self.commit_changes(change_type))
+
+    async def commit_changes(self, change_type):
+        """异步提交变更到 Hugging Face Hub，带重试机制"""
+        max_retries = 3
+        retry_delay = 5
+        change_summary = f"{change_type} ({len(self.pending_changes)} 文件)"
+        self.pending_changes = []  # 清空缓冲区
+        for attempt in range(max_retries):
+            try:
+                commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
+                logger.info(f"开始上传: {commit_message}")
+                await asyncio.to_thread(self.api.upload_folder,
+                    folder_path=self.data_directory,
+                    repo_id=self.repo_id,
+                    repo_type="dataset",
+                    commit_message=commit_message,
+                    ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
+                )
+                logger.info(f"✅ 成功提交变更到 Hugging Face Hub: {commit_message}")
+                return
+            except Exception as e:
+                logger.error(f"❌ 提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
+                if attempt < max_retries - 1:
+                    logger.info(f"将在 {retry_delay} 秒后重试...")
+                    await asyncio.sleep(retry_delay)
+        logger.error(f"❌ 达到最大重试次数，上传失败")
+
+def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
+    """启动目录监控服务"""
+    # 硬编码的 base64 编码的 HF_TOKEN
+    if not hf_token:
+        hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
+        hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
+        logger.info(f"HF_TOKEN value: {hf_token}")
+    
+    if not repo_id:
+        raise ValueError("必须提供 repo_id 参数")
+    
+    if not os.path.exists(data_directory):
+        logger.warning(f"监控目录 {data_directory} 不存在，正在创建...")
+        os.makedirs(data_directory, exist_ok=True)
+    
+    event_handler = DataDirectoryHandler(
+        repo_id=repo_id,
+        hf_token=hf_token,
+        data_directory=data_directory
+    )
+    
+    observer = Observer()
+    observer.schedule(event_handler, data_directory, recursive=True)
+    observer.start()
+    logger.info(f"🎯 目录监控服务已启动: {data_directory}")
+    
+    return observer
+
+def start_gitea_server(port=7860):
+    """启动 Gitea 服务器"""
+    def run_gitea():
+        try:
+            gitea_process = subprocess.Popen(
+                ['gitea', 'web', '--port', str(port)],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                universal_newlines=True
+            )
+            logger.info(f"🚀 Gitea 服务器已启动，端口: {port}")
+            logger.info(f"📊 访问地址: http://localhost:{port}")
+            
+            while True:
+                output = gitea_process.stdout.readline()
+                if output == '' and gitea_process.poll() is not None:
+                    break
+                if output:
+                    logger.info(f"Gitea: {output.strip()}")
+            
+            return_code = gitea_process.poll()
+            if return_code != 0:
+                error_output = gitea_process.stderr.read()
+                logger.error(f"❌ Gitea 服务器异常退出，返回码: {return_code}")
+                logger.error(f"错误信息: {error_output}")
+            else:
+                logger.info("✅ Gitea 服务器正常退出")
+                
+        except FileNotFoundError:
+            logger.error("❌ 未找到 gitea 命令，请确保 Gitea 已正确安装")
+        except Exception as e:
+            logger.error(f"❌ 启动 Gitea 服务器时发生错误: {e}")
+    
+    gitea_thread = threading.Thread(target=run_gitea)
+    gitea_thread.daemon = True
+    gitea_thread.start()
+    
+    return gitea_thread
+
+async def main():
+    """主函数 - 启动 Gitea 和目录监控服务"""
+    # 配置参数
+    CONFIG = {
+        "data_directory": "/data",
+        "repo_id": os.getenv("REPO_ID", "02engine/02gitea"),
+        "hf_token": os.getenv('HF_TOKEN'),
+        "gitea_port": 7860
+    }
+    
+    logger.info("🚀 启动集成服务...")
+    
+    try:
+        # 先运行 pullhf.py 拉取最新数据集
+        logger.info("运行 pullhf.py 拉取最新数据集")
+        pull_result = subprocess.run(["python3", "/pullhf.py"], check=True)
+        if pull_result.returncode != 0:
+            logger.error("❌ pullhf.py 执行失败，退出")
+            exit(1)
+        
+        # 启动 Gitea 服务器
+        gitea_thread = start_gitea_server(CONFIG["gitea_port"])
+        
+        # 启动目录监控服务
+        observer = start_directory_monitoring(
+            data_directory=CONFIG["data_directory"],
+            repo_id=CONFIG["repo_id"],
+            hf_token=CONFIG["hf_token"]
+        )
+        
+        logger.info("✅ 所有服务已启动完成！")
+        logger.info("📁 目录监控: /data → Hugging Face Hub")
+        logger.info("🌐 Gitea 服务: http://localhost:7860")
+        logger.info("🛑 按 Ctrl+C 停止所有服务")
+        
+        try:
+            while True:
+                await asyncio.sleep(1)
+        except KeyboardInterrupt:
+            logger.info("正在停止服务...")
+            
+    except Exception as e:
+        logger.error(f"❌ 启动服务时发生错误: {e}")
+    finally:
+        if 'observer' in locals():
+            observer.stop()
+            observer.join()
+        logger.info("所有服务已停止")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+EOF
+
+# 设置工作目录
+WORKDIR /data
+
+# 暴露 Spaces 默认端口（文档用途，实际由 $PORT 控制）
+EXPOSE 7860
+
+# Gitea 环境变量：适配 Spaces 端口，禁用 SSH，动态决定是否跳过安装页面
+RUN echo '#!/bin/sh' > /set_gitea_env.sh && \
+    echo 'if [ -f /data/gitea_skip_install ]; then' >> /set_gitea_env.sh && \
+    echo '  export GITEA__server__STARTUP_DISABLE_INSTALL_FORM=true' >> /set_gitea_env.sh && \
+    echo 'else' >> /set_gitea_env.sh && \
+    echo '  export GITEA__server__STARTUP_DISABLE_INSTALL_FORM=false' >> /set_gitea_env.sh && \
+    echo 'fi' >> /set_gitea_env.sh && \
+    echo 'exec "$@"' >> /set_gitea_env.sh && \
+    chmod +x /set_gitea_env.sh
+
+ENV GITEA__database__DB_TYPE=sqlite3 \
+    GITEA__database__PATH=/data/gitea/gitea.db \
+    GITEA__server__ROOT_URL=https://deep-sea-02gitea.hf.space/ \
+    GITEA__server__HTTP_ADDR=0.0.0.0 \
+    GITEA__server__HTTP_PORT=7860 \
+    GITEA__server__DISABLE_SSH=true \
+    USER=git
+
+# 切换到 git 用户（非 root）
+USER git
+
+# 运行 Gitea，显式指定 Spaces 默认端口
+RUN cat /uploadhf.py
+CMD ["/set_gitea_env.sh", "python3", "/uploadhf.py"]