# ================ GitLab CE + HF 双向同步 + 调试清空模式 ================ FROM gitlab/gitlab-ce:17.5.2-ce.0 # ---------- 以 root 安装依赖 ---------- USER root RUN apt-get update && \ apt-get install -y --no-install-recommends \ python3 python3-pip python3-venv git ca-certificates tzdata && \ pip3 install --break-system-packages --upgrade pip && \ pip3 install --break-system-packages watchdog huggingface_hub aiohttp pytz && \ rm -rf /var/lib/apt/lists/* # ---------- 创建持久化目录 ---------- RUN mkdir -p /data/gitlab/config /data/gitlab/logs /data/gitlab/data && \ chown -R git:git /data && \ chmod -R 770 /data # ===================== pullhf.py(支持 CLEAR_DATA_ON_START 调试模式)===================== COPY --chown=root:root <<'EOF' /pullhf.py import os import shutil import base64 import logging from datetime import datetime import pytz from huggingface_hub import snapshot_download beijing_tz = pytz.timezone('Asia/Shanghai') logging.basicConfig( level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) logger = logging.getLogger(__name__) class BeijingFormatter(logging.Formatter): def formatTime(self, record, datefmt=None): dt = datetime.fromtimestamp(record.created, tz=beijing_tz) return dt.strftime(datefmt or '%Y-%m-%d %H:%M:%S') for h in logging.getLogger().handlers: h.setFormatter(BeijingFormatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M:%S')) def pull_or_clear(): clear_mode = os.getenv("CLEAR_DATA_ON_START", "0").lower() in ("1", "true", "yes") target = "/data" if clear_mode: logger.info("CLEAR_DATA_ON_START=1 已启用,正在强制清空 /data 目录(调试模式)") if os.path.exists(target): shutil.rmtree(target) os.makedirs(target, exist_ok=True) # 故意不创建 initialized 标记 → 强制走首次安装页面 logger.info("/data 已清空,GitLab 将显示首次安装页面(请设置 root 密码)") return # 正常模式:从 HF Dataset 拉取数据 hf_token_b64 = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ==" hf_token = base64.b64decode(hf_token_b64).decode('utf-8') repo_id = os.getenv("REPO_ID", "02engine/02gitea") temp_dir = "/tmp/hf_download" if os.path.exists(temp_dir): shutil.rmtree(temp_dir, ignore_errors=True) logger.info(f"正在从 Hugging Face Dataset {repo_id} 拉取最新数据...") snapshot_download( repo_id=repo_id, repo_type="dataset", local_dir=temp_dir, token=hf_token, ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"] ) if os.path.exists(target): shutil.rmtree(target) os.makedirs(target, exist_ok=True) for item in os.listdir(temp_dir): shutil.move(os.path.join(temp_dir, item), os.path.join(target, item)) shutil.rmtree(temp_dir, ignore_errors=True) # 创建标记:已初始化,跳过首次密码设置 with open("/data/gitlab_initialized", "w") as f: f.write("true") os.system("chown -R git:git /data && chmod -R 770 /data") logger.info("数据拉取完成,GitLab 将直接进入已配置状态(无需设置 root 密码)") if __name__ == "__main__": pull_or_clear() EOF # ===================== uploadhf.py(实时同步到 HF)===================== COPY --chown=git:git <<'EOF' /uploadhf.py import os import time import base64 import logging import asyncio import threading import subprocess from pathlib import Path from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler from huggingface_hub import HfApi, utils import pytz from datetime import datetime utils.disable_progress_bars() beijing_tz = pytz.timezone('Asia/Shanghai') logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class BeijingFormatter(logging.Formatter): def formatTime(self, record, datefmt=None): dt = datetime.fromtimestamp(record.created, tz=beijing_tz) return dt.strftime(datefmt or '%Y-%m-%d %H:%M:%S') for h in logging.getLogger().handlers: h.setFormatter(BeijingFormatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M:%S')) class HFHandler(FileSystemEventHandler): def __init__(self, repo_id, token): self.repo_id = repo_id self.api = HfApi(token=token) self.last = 0 self.delay = 3 def on_any_event(self, event): if event.is_directory or event.src_path.endswith(('.tmp', '.log', '.lock')): return now = time.time() if now - self.last > self.delay: self.last = now asyncio.run(self.commit()) async def commit(self): try: msg = f"Auto sync from HF Space {time.strftime('%Y-%m-%d %H:%M:%S')}" logger.info(f"正在上传变更 → {msg}") await asyncio.to_thread( self.api.upload_folder, folder_path="/data", repo_id=self.repo_id, repo_type="dataset", commit_message=msg, ignore_patterns=["*.tmp", "*.log", "*.lock", ".git/*"] ) logger.info("上传成功") except Exception as e: logger.error(f"上传失败: {e}") def start_monitor(): token_b64 = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ==" token = base64.b64decode(token_b64).decode() repo_id = os.getenv("REPO_ID", "02engine/02gitea") handler = HFHandler(repo_id, token) observer = Observer() observer.schedule(handler, "/data", recursive=True) observer.start() logger.info(f"已启动 /data → HF Dataset {repo_id} 实时同步") return observer def start_gitlab(): logger.info("执行 gitlab-ctl reconfigure ...") subprocess.run(["gitlab-ctl", "reconfigure"], check=True) logger.info("启动 GitLab 服务(gitlab-ctl tail 会保持前台运行)") subprocess.run(["gitlab-ctl", "tail"]) if __name__ == "__main__": # 1. 拉取或清空数据 subprocess.run(["python3", "/pullhf.py"], check=False) # 2. 启动监控(实时推送到 HF) observer = start_monitor() # 3. 启动 GitLab gitlab_thread = threading.Thread(target=start_gitlab, daemon=True) gitlab_thread.start() logger.info("GitLab + HF 双向同步服务已全部启动!") logger.info("访问地址: http://localhost:7860 (HF Space 会自动映射 $PORT)") logger.info("调试模式:设置 CLEAR_DATA_ON_START=1 可强制清空数据并显示首次安装页面") try: while True: time.sleep(1) except KeyboardInterrupt: observer.stop() observer.join() EOF # ===================== GitLab 配置(兼容 HF Spaces)===================== ENV GITLAB_OMNIBUS_CONFIG=\ external_url 'https://deep-sea-02gitea.hf.space/'; \ nginx['listen_port'] = 7860; \ nginx['listen_https'] = false; # 如果检测到已初始化标记,则跳过首次 root 密码设置 RUN echo '#!/bin/bash\n\ if [ -f /data/gitlab_initialized ]; then\n\ echo "检测到已初始化,自动跳过首次密码设置页面"\n\ export GITLAB_ROOT_PASSWORD="skip-initial-setup-$(date +%s)"\n\ fi\n\ exec "$@"\n' > /etc/gitlab/gitlab.rb.prepend.sh && \ chmod +x /etc/gitlab/gitlab.rb.prepend.sh # ===================== 启动 ===================== EXPOSE 7860 ENTRYPOINT ["/etc/gitlab/gitlab.rb.prepend.sh", "/assets/wrapper"] CMD ["python3", "/uploadhf.py"]