我承认豆包就是个猴子

Dockerfile

@@ -1,351 +1,214 @@
-FROM gitlab/gitlab-ce:17.6.0-ce.0
+# ================ GitLab CE + HF 双向同步 + 调试清空模式 ================
+FROM gitlab/gitlab-ce:17.5.2-ce.0
 
-# 以 root 用户执行系统级操作
+# ---------- 以 root 安装依赖 ----------
 USER root
 
-# 安装 Python、pip 及所需依赖库
 RUN apt-get update && \
-    apt-get install -y python3 python3-pip git && \
-    pip3 install --break-system-packages --upgrade pip
-
-# 安装 Python 包
-RUN pip3 install --break-system-packages watchdog huggingface_hub aiohttp pytz
-
-# 预创建数据目录并设置权限（适配 GitLab 目录规范）
-RUN mkdir -p /data/gitlab /data/hf_sync && \
+    apt-get install -y --no-install-recommends \
+        python3 python3-pip python3-venv git ca-certificates tzdata && \
+    pip3 install --break-system-packages --upgrade pip && \
+    pip3 install --break-system-packages watchdog huggingface_hub aiohttp pytz && \
+    rm -rf /var/lib/apt/lists/*
+
+# ---------- 创建持久化目录 ----------
+RUN mkdir -p /data/gitlab/config /data/gitlab/logs /data/gitlab/data && \
     chown -R git:git /data && \
     chmod -R 770 /data
 
-# 复制拉取 Hugging Face 数据集的脚本（移除 Gitea 专属的跳过初始化逻辑）
+# ===================== pullhf.py（支持 CLEAR_DATA_ON_START 调试模式）=====================
 COPY --chown=root:root <<'EOF' /pullhf.py
 import os
 import shutil
-from huggingface_hub import snapshot_download
-import logging
 import base64
+import logging
 from datetime import datetime
 import pytz
+from huggingface_hub import snapshot_download
 
-# 配置日志，使用北京时间
 beijing_tz = pytz.timezone('Asia/Shanghai')
 logging.basicConfig(
     level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    datefmt='%Y-%m-%d %H:%M:%S %Z'
+    format='%(asctime)s %(levelname)s %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
 )
 logger = logging.getLogger(__name__)
 
-# 自定义日志格式化器以使用北京时间
-class BeijingTimeFormatter(logging.Formatter):
+class BeijingFormatter(logging.Formatter):
     def formatTime(self, record, datefmt=None):
         dt = datetime.fromtimestamp(record.created, tz=beijing_tz)
-        if datefmt:
-            return dt.strftime(datefmt)
-        return dt.strftime("%Y-%m-%d %H:%M:%S %Z")
-
-# 应用自定义格式化器
-for handler in logging.getLogger().handlers:
-    handler.setFormatter(BeijingTimeFormatter(
-        fmt='%(asctime)s - %(levelname)s - %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S %Z'
-    ))
-
-def pull_from_hf_hub(repo_id, data_directory="/data/gitlab"):
-    """从 Hugging Face Hub 数据集仓库拉取数据到 GitLab 数据目录"""
-    # 硬编码的 base64 编码的 HF_TOKEN
-    hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
-    hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
-    
-    try:
-        logger.info(f"HF_TOKEN value: {hf_token}")
-        # 临时目录用于下载
-        temp_dir = "/tmp/hf_download"
-        if os.path.exists(temp_dir):
-            shutil.rmtree(temp_dir, ignore_errors=True)
-        
-        # 下载 Hugging Face 数据集
-        logger.info(f"正在从 Hugging Face Hub 拉取数据集: {repo_id}")
-        snapshot_download(
-            repo_id=repo_id,
-            repo_type="dataset",
-            local_dir=temp_dir,
-            token=hf_token,
-            ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
-        )
-
-        # 清空现有的目标目录
-        if os.path.exists(data_directory):
-            logger.info(f"正在清空现有目录: {data_directory}")
-            shutil.rmtree(data_directory, ignore_errors=True)
-        
-        # 创建目录并移动下载的内容
-        os.makedirs(data_directory, exist_ok=True)
-        for item in os.listdir(temp_dir):
-            src = os.path.join(temp_dir, item)
-            dst = os.path.join(data_directory, item)
-            shutil.move(src, dst)
-        
-        # 清理临时目录
+        return dt.strftime(datefmt or '%Y-%m-%d %H:%M:%S')
+
+for h in logging.getLogger().handlers:
+    h.setFormatter(BeijingFormatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M:%S'))
+
+def pull_or_clear():
+    clear_mode = os.getenv("CLEAR_DATA_ON_START", "0").lower() in ("1", "true", "yes")
+    target = "/data"
+
+    if clear_mode:
+        logger.info("CLEAR_DATA_ON_START=1 已启用，正在强制清空 /data 目录（调试模式）")
+        if os.path.exists(target):
+            shutil.rmtree(target)
+        os.makedirs(target, exist_ok=True)
+        # 故意不创建 initialized 标记 → 强制走首次安装页面
+        logger.info("/data 已清空，GitLab 将显示首次安装页面（请设置 root 密码）")
+        return
+
+    # 正常模式：从 HF Dataset 拉取数据
+    hf_token_b64 = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
+    hf_token = base64.b64decode(hf_token_b64).decode('utf-8')
+    repo_id = os.getenv("REPO_ID", "02engine/02gitea")
+
+    temp_dir = "/tmp/hf_download"
+    if os.path.exists(temp_dir):
         shutil.rmtree(temp_dir, ignore_errors=True)
-        
-        # 修复权限：确保目录为 git:git 所有
-        logger.info("修复数据目录权限")
-        os.system("chown -R git:git /data")
-        os.system("chmod -R 770 /data")
-        
-        # 调试：列出目录结构
-        logger.info("调试：列出 /data 目录结构和权限")
-        os.system("ls -la /data")
-        os.system("ls -la /data/gitlab 2>/dev/null || echo '/data/gitlab 不存在'")
-        
-        logger.info(f"成功从 Hugging Face Hub 拉取数据到 {data_directory}")
-        return True
-
-    except Exception as e:
-        logger.error(f"拉取 Hugging Face 数据集失败: {e}")
-        return False
+
+    logger.info(f"正在从 Hugging Face Dataset {repo_id} 拉取最新数据...")
+    snapshot_download(
+        repo_id=repo_id,
+        repo_type="dataset",
+        local_dir=temp_dir,
+        token=hf_token,
+        ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
+    )
+
+    if os.path.exists(target):
+        shutil.rmtree(target)
+    os.makedirs(target, exist_ok=True)
+    for item in os.listdir(temp_dir):
+        shutil.move(os.path.join(temp_dir, item), os.path.join(target, item))
+    shutil.rmtree(temp_dir, ignore_errors=True)
+
+    # 创建标记：已初始化，跳过首次密码设置
+    with open("/data/gitlab_initialized", "w") as f:
+        f.write("true")
+
+    os.system("chown -R git:git /data && chmod -R 770 /data")
+    logger.info("数据拉取完成，GitLab 将直接进入已配置状态（无需设置 root 密码）")
 
 if __name__ == "__main__":
-    repo_id = os.getenv("REPO_ID", "02engine/02gitea")
-    if not pull_from_hf_hub(repo_id):
-        logger.error("拉取数据集失败，退出")
-        exit(1)
+    pull_or_clear()
 EOF
 
-# 复制目录监控及上传 Hugging Face 的脚本（移除 Gitea 相关逻辑）
+# ===================== uploadhf.py（实时同步到 HF）=====================
 COPY --chown=git:git <<'EOF' /uploadhf.py
 import os
 import time
+import base64
 import logging
+import asyncio
 import threading
 import subprocess
-import asyncio
-import aiohttp
 from pathlib import Path
 from watchdog.observers import Observer
 from watchdog.events import FileSystemEventHandler
-from huggingface_hub import HfApi
-from huggingface_hub import utils
-import base64
-from datetime import datetime
+from huggingface_hub import HfApi, utils
 import pytz
+from datetime import datetime
 
-# 全局禁用所有 Hugging Face 进度条
 utils.disable_progress_bars()
 
-# 配置日志，使用北京时间
 beijing_tz = pytz.timezone('Asia/Shanghai')
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    datefmt='%Y-%m-%d %H:%M:%S %Z'
-)
+logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-# 自定义日志格式化器
-class BeijingTimeFormatter(logging.Formatter):
+class BeijingFormatter(logging.Formatter):
     def formatTime(self, record, datefmt=None):
         dt = datetime.fromtimestamp(record.created, tz=beijing_tz)
-        if datefmt:
-            return dt.strftime(datefmt)
-        return dt.strftime("%Y-%m-%d %H:%M:%S %Z")
-
-for handler in logging.getLogger().handlers:
-    handler.setFormatter(BeijingTimeFormatter(
-        fmt='%(asctime)s - %(levelname)s - %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S %Z'
-    ))
-
-class DataDirectoryHandler(FileSystemEventHandler):
-    def __init__(self, repo_id, hf_token, data_directory="/data/gitlab"):
+        return dt.strftime(datefmt or '%Y-%m-%d %H:%M:%S')
+for h in logging.getLogger().handlers:
+    h.setFormatter(BeijingFormatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M:%S'))
+
+class HFHandler(FileSystemEventHandler):
+    def __init__(self, repo_id, token):
         self.repo_id = repo_id
-        self.hf_token = hf_token
-        self.data_directory = data_directory
-        self.api = HfApi(token=hf_token)
-        self.last_commit_time = 0
-        self.commit_delay = 1
-        self.pending_changes = []
-        logger.info(f"初始化监控器，监控目录: {data_directory}，目标仓库: {repo_id}")
-    
+        self.api = HfApi(token=token)
+        self.last = 0
+        self.delay = 3
+
     def on_any_event(self, event):
-        if event.is_directory:
+        if event.is_directory or event.src_path.endswith(('.tmp', '.log', '.lock')):
             return
-        self.pending_changes.append((event.event_type, event.src_path))
-        self.schedule_commit(f"文件{event.event_type}")
-
-    def schedule_commit(self, change_type):
-        current_time = time.time()
-        if current_time - self.last_commit_time > self.commit_delay:
-            self.last_commit_time = current_time
-            asyncio.run(self.commit_changes(change_type))
-
-    async def commit_changes(self, change_type):
-        max_retries = 3
-        retry_delay = 5
-        change_summary = f"{change_type} ({len(self.pending_changes)} 文件)"
-        self.pending_changes = []
-        
-        for attempt in range(max_retries):
-            try:
-                commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
-                logger.info(f"开始上传: {commit_message}")
-
-                await asyncio.to_thread(
-                    self.api.upload_folder,
-                    folder_path=self.data_directory,
-                    repo_id=self.repo_id,
-                    repo_type="dataset",
-                    commit_message=commit_message,
-                    ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
-                )
-
-                logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
-                return
-
-            except Exception as e:
-                logger.error(f"提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
-                if attempt < max_retries - 1:
-                    logger.info(f"将在 {retry_delay} 秒后重试...")
-                    await asyncio.sleep(retry_delay)
-        
-        logger.error(f"达到最大重试次数，上传失败")
-
-def start_directory_monitoring(data_directory="/data/gitlab", repo_id=None, hf_token=None):
-    if not hf_token:
-        hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
-        hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
-        logger.info(f"HF_TOKEN 已解码（长度: {len(hf_token)}）")
-    
-    if not repo_id:
-        raise ValueError("必须提供 repo_id 参数")
-    
-    if not os.path.exists(data_directory):
-        logger.warning(f"监控目录 {data_directory} 不存在，正在创建...")
-        os.makedirs(data_directory, exist_ok=True)
-    
-    event_handler = DataDirectoryHandler(
-        repo_id=repo_id,
-        hf_token=hf_token,
-        data_directory=data_directory
-    )
-    
-    observer = Observer()
-    observer.schedule(event_handler, data_directory, recursive=True)
-    observer.start()
-    logger.info(f"目录监控服务已启动: {data_directory}")
-    
-    return observer
+        now = time.time()
+        if now - self.last > self.delay:
+            self.last = now
+            asyncio.run(self.commit())
 
-def start_gitlab_server():
-    def run_gitlab():
-        try:
-            gitlab_process = subprocess.Popen(
-                ['/assets/wrapper'],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                universal_newlines=True
-            )
-            logger.info("GitLab 服务已启动")
-            
-            while True:
-                output = gitlab_process.stdout.readline()
-                if output == '' and gitlab_process.poll() is not None:
-                    break
-                if output:
-                    logger.info(f"GitLab: {output.strip()}")
-            
-            return_code = gitlab_process.poll()
-            if return_code != 0:
-                error_output = gitlab_process.stderr.read()
-                logger.error(f"GitLab 服务器异常退出，返回码: {return_code}")
-                logger.error(f"错误信息: {error_output}")
-            else:
-                logger.info("GitLab 服务器正常退出")
-                
-        except Exception as e:
-            logger.error(f"启动 GitLab 服务器时发生错误: {e}")
-    
-    gitlab_thread = threading.Thread(target=run_gitlab)
-    gitlab_thread.daemon = True
-    gitlab_thread.start()
-    
-    return gitlab_thread
-
-async def main():
-    CONFIG = {
-        "data_directory": "/data/gitlab",
-        "repo_id": os.getenv("REPO_ID", "02engine/02gitea"),
-        "hf_token": os.getenv('HF_TOKEN'),
-    }
-    
-    max_retries = 5
-    retry_count = 0
-    
-    while retry_count < max_retries:
+    async def commit(self):
         try:
-            logger.info(f"启动集成服务 (尝试 {retry_count + 1}/{max_retries})...")
-            
-            logger.info("运行 pullhf.py 拉取最新数据集")
-            pull_result = subprocess.run(["python3", "/pullhf.py"], check=True)
-            if pull_result.returncode != 0:
-                logger.error("pullhf.py 执行失败，退出")
-                exit(1)
-            
-            gitlab_thread = start_gitlab_server()
-            observer = start_directory_monitoring(
-                data_directory=CONFIG["data_directory"],
-                repo_id=CONFIG["repo_id"],
-                hf_token=CONFIG["hf_token"]
+            msg = f"Auto sync from HF Space {time.strftime('%Y-%m-%d %H:%M:%S')}"
+            logger.info(f"正在上传变更 → {msg}")
+            await asyncio.to_thread(
+                self.api.upload_folder,
+                folder_path="/data",
+                repo_id=self.repo_id,
+                repo_type="dataset",
+                commit_message=msg,
+                ignore_patterns=["*.tmp", "*.log", "*.lock", ".git/*"]
             )
-            
-            logger.info("所有服务已启动完成！")
-            logger.info("目录监控: /data/gitlab to Hugging Face Hub")
-            logger.info("GitLab 服务已启动，可通过对应端口访问")
-            logger.info("按 Ctrl+C 停止所有服务")
-            
-            try:
-                while True:
-                    await asyncio.sleep(1)
-            except KeyboardInterrupt:
-                logger.info("正在停止服务...")
-                break
-            
+            logger.info("上传成功")
         except Exception as e:
-            retry_count += 1
-            logger.error(f"服务崩溃 (尝试 {retry_count}/{max_retries}): {e}")
-            if retry_count < max_retries:
-                logger.info(f"将在 5 秒后尝试重启...")
-                await asyncio.sleep(5)
-            else:
-                logger.error(f"达到最大重试次数 ({max_retries})，退出")
-                exit(1)
-        finally:
-            if 'observer' in locals():
-                observer.stop()
-                observer.join()
-            logger.info("所有服务已停止")
+            logger.error(f"上传失败: {e}")
+
+def start_monitor():
+    token_b64 = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
+    token = base64.b64decode(token_b64).decode()
+    repo_id = os.getenv("REPO_ID", "02engine/02gitea")
+
+    handler = HFHandler(repo_id, token)
+    observer = Observer()
+    observer.schedule(handler, "/data", recursive=True)
+    observer.start()
+    logger.info(f"已启动 /data → HF Dataset {repo_id} 实时同步")
+    return observer
+
+def start_gitlab():
+    logger.info("执行 gitlab-ctl reconfigure ...")
+    subprocess.run(["gitlab-ctl", "reconfigure"], check=True)
+    logger.info("启动 GitLab 服务（gitlab-ctl tail 会保持前台运行）")
+    subprocess.run(["gitlab-ctl", "tail"])
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    # 1. 拉取或清空数据
+    subprocess.run(["python3", "/pullhf.py"], check=False)
+
+    # 2. 启动监控（实时推送到 HF）
+    observer = start_monitor()
+
+    # 3. 启动 GitLab
+    gitlab_thread = threading.Thread(target=start_gitlab, daemon=True)
+    gitlab_thread.start()
+
+    logger.info("GitLab + HF 双向同步服务已全部启动！")
+    logger.info("访问地址: http://localhost:7860 （HF Space 会自动映射 $PORT）")
+    logger.info("调试模式：设置 CLEAR_DATA_ON_START=1 可强制清空数据并显示首次安装页面")
+
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+        observer.join()
 EOF
 
-# 配置 GitLab 环境变量（基础必要配置）
-ENV GITLAB_OMNIBUS_CONFIG="external_url 'https://deep-sea-02gitlab.hf.space'; \
-    gitlab_rails['time_zone'] = 'Asia/Shanghai'; \
-    gitlab_rails['db_adapter'] = 'postgresql'; \
-    gitlab_rails['db_database'] = 'gitlabhq_production'; \
-    gitlab_rails['db_username'] = 'gitlab'; \
-    gitlab_rails['db_password'] = 'gitlabdbpass'; \
-    gitlab_rails['db_host'] = 'localhost'; \
-    gitlab_rails['db_port'] = 5432; \
-    unicorn['port'] = 8080; \
-    nginx['listen_port'] = 7860;"
-
-# 暴露 GitLab 服务端口（适配 Spaces 默认端口）
-EXPOSE 7860 22 8080
-
-# 切换到 git 用户
-USER git
-
-# 启动同步脚本并托管 GitLab 服务
+# ===================== GitLab 配置（兼容 HF Spaces）=====================
+ENV GITLAB_OMNIBUS_CONFIG=\
+external_url 'https://deep-sea-02gitea.hf.space/'; \
+nginx['listen_port'] = 7860; \
+nginx['listen_https'] = false;
+
+# 如果检测到已初始化标记，则跳过首次 root 密码设置
+RUN echo '#!/bin/bash\n\
+if [ -f /data/gitlab_initialized ]; then\n\
+  echo "检测到已初始化，自动跳过首次密码设置页面"\n\
+  export GITLAB_ROOT_PASSWORD="skip-initial-setup-$(date +%s)"\n\
+fi\n\
+exec "$@"\n' > /etc/gitlab/gitlab.rb.prepend.sh && \
+    chmod +x /etc/gitlab/gitlab.rb.prepend.sh
+
+# ===================== 启动 =====================
+EXPOSE 7860
+
+ENTRYPOINT ["/etc/gitlab/gitlab.rb.prepend.sh", "/assets/wrapper"]
 CMD ["python3", "/uploadhf.py"]