Update Dockerfile
Browse files- Dockerfile +75 -111
Dockerfile
CHANGED
|
@@ -1,14 +1,14 @@
|
|
| 1 |
FROM gitea/gitea:1.24.6
|
| 2 |
|
| 3 |
-
# 以 root 用户设置权限(Spaces 允许构建时
|
| 4 |
USER root
|
| 5 |
|
| 6 |
-
# 安装 Python、pip、git
|
| 7 |
RUN apk update && \
|
| 8 |
-
apk add python3 py3-pip git
|
| 9 |
pip3 install --break-system-packages --upgrade pip
|
| 10 |
|
| 11 |
-
# 安装 Python 包,包括 pytz
|
| 12 |
RUN pip3 install --break-system-packages watchdog huggingface_hub aiohttp pytz
|
| 13 |
|
| 14 |
# 预创建 Gitea 所需目录并设置权限
|
|
@@ -23,14 +23,33 @@ import shutil
|
|
| 23 |
from huggingface_hub import snapshot_download
|
| 24 |
import logging
|
| 25 |
import base64
|
|
|
|
|
|
|
| 26 |
|
| 27 |
-
# 配置日志
|
|
|
|
| 28 |
logging.basicConfig(
|
| 29 |
level=logging.INFO,
|
| 30 |
-
format='%(asctime)s - %(levelname)s - %(message)s'
|
|
|
|
| 31 |
)
|
| 32 |
logger = logging.getLogger(__name__)
|
| 33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
def pull_from_hf_hub(repo_id, data_directory="/data"):
|
| 35 |
"""从 Hugging Face Hub 数据集仓库拉取数据替换 /data 目录"""
|
| 36 |
# 硬编码的 base64 编码的 HF_TOKEN
|
|
@@ -109,7 +128,7 @@ if __name__ == "__main__":
|
|
| 109 |
exit(1)
|
| 110 |
EOF
|
| 111 |
|
| 112 |
-
# 复制上传脚本,
|
| 113 |
COPY --chown=git:git <<'EOF' /uploadhf.py
|
| 114 |
import os
|
| 115 |
import time
|
|
@@ -126,20 +145,30 @@ import base64
|
|
| 126 |
from datetime import datetime
|
| 127 |
import pytz
|
| 128 |
|
| 129 |
-
# 配置日志,使用北京时间
|
| 130 |
beijing_tz = pytz.timezone('Asia/Shanghai')
|
| 131 |
logging.basicConfig(
|
| 132 |
level=logging.INFO,
|
| 133 |
format='%(asctime)s - %(levelname)s - %(message)s',
|
| 134 |
-
datefmt='%Y-%m-%d %H:%M:%S %Z'
|
| 135 |
-
handlers=[
|
| 136 |
-
logging.StreamHandler()
|
| 137 |
-
]
|
| 138 |
)
|
| 139 |
-
# 设置日志的时区
|
| 140 |
-
logging.Formatter.converter = lambda *args: datetime.now(beijing_tz).timetuple()
|
| 141 |
logger = logging.getLogger(__name__)
|
| 142 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
class DataDirectoryHandler(FileSystemEventHandler):
|
| 144 |
"""处理 /data 目录文件变化的监控器"""
|
| 145 |
|
|
@@ -151,7 +180,6 @@ class DataDirectoryHandler(FileSystemEventHandler):
|
|
| 151 |
self.last_commit_time = 0
|
| 152 |
self.commit_delay = 1 # 防抖延迟 1 秒
|
| 153 |
self.pending_changes = [] # 缓冲待上传变更
|
| 154 |
-
self.loop = asyncio.get_event_loop() # 获取主事件循环
|
| 155 |
logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
|
| 156 |
|
| 157 |
def on_any_event(self, event):
|
|
@@ -166,8 +194,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
|
|
| 166 |
current_time = time.time()
|
| 167 |
if current_time - self.last_commit_time > self.commit_delay:
|
| 168 |
self.last_commit_time = current_time
|
| 169 |
-
|
| 170 |
-
self.loop.call_soon_threadsafe(lambda: asyncio.create_task(self.commit_changes(change_type)))
|
| 171 |
|
| 172 |
async def commit_changes(self, change_type):
|
| 173 |
"""异步提交变更到 Hugging Face Hub,带重试机制"""
|
|
@@ -177,7 +204,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
|
|
| 177 |
self.pending_changes = [] # 清空缓冲区
|
| 178 |
for attempt in range(max_retries):
|
| 179 |
try:
|
| 180 |
-
commit_message = f"自动提交: {change_summary} - {
|
| 181 |
logger.info(f"开始上传: {commit_message}")
|
| 182 |
await asyncio.to_thread(self.api.upload_folder,
|
| 183 |
folder_path=self.data_directory,
|
|
@@ -204,8 +231,7 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
|
|
| 204 |
logger.info(f"HF_TOKEN value: {hf_token}")
|
| 205 |
|
| 206 |
if not repo_id:
|
| 207 |
-
|
| 208 |
-
exit(1)
|
| 209 |
|
| 210 |
if not os.path.exists(data_directory):
|
| 211 |
logger.warning(f"监控目录 {data_directory} 不存在,正在创建...")
|
|
@@ -225,98 +251,37 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
|
|
| 225 |
return observer
|
| 226 |
|
| 227 |
def start_gitea_server(port=7860):
|
| 228 |
-
"""启动 Gitea 服务器
|
| 229 |
def run_gitea():
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
gitea_process = subprocess.Popen(
|
| 244 |
-
['gitea', 'web', '--port', str(port)],
|
| 245 |
-
stdout=subprocess.PIPE,
|
| 246 |
-
stderr=subprocess.PIPE,
|
| 247 |
-
universal_newlines=True
|
| 248 |
-
)
|
| 249 |
-
logger.info(f"🚀 Gitea 服务器启动尝试 {attempt + 1}/{max_restart_attempts},端口: {port}")
|
| 250 |
-
logger.info(f"📊 访问地址: http://localhost:{port}")
|
| 251 |
-
|
| 252 |
-
# 健康检查:等待 Gitea 响应
|
| 253 |
-
start_time = time.time()
|
| 254 |
-
health_check_passed = False
|
| 255 |
-
while time.time() - start_time < health_check_timeout:
|
| 256 |
-
try:
|
| 257 |
-
result = subprocess.run(
|
| 258 |
-
['curl', '-s', '-f', f'http://localhost:{port}/'],
|
| 259 |
-
capture_output=True, text=True, timeout=5
|
| 260 |
-
)
|
| 261 |
-
if result.returncode == 0:
|
| 262 |
-
logger.info("✅ Gitea 健康检查通过")
|
| 263 |
-
health_check_passed = True
|
| 264 |
-
break
|
| 265 |
-
except subprocess.SubprocessError:
|
| 266 |
-
pass
|
| 267 |
-
time.sleep(1)
|
| 268 |
-
|
| 269 |
-
if not health_check_passed:
|
| 270 |
-
logger.error("❌ Gitea 健康检查失败,服务未就绪")
|
| 271 |
-
gitea_process.terminate()
|
| 272 |
-
error_output = gitea_process.stderr.read()
|
| 273 |
-
if error_output:
|
| 274 |
-
logger.error(f"错误信息: {error_output.strip()}")
|
| 275 |
-
attempt += 1
|
| 276 |
-
if attempt < max_restart_attempts:
|
| 277 |
-
logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
|
| 278 |
-
time.sleep(restart_delay)
|
| 279 |
-
else:
|
| 280 |
-
logger.error(f"❌ 达到最大重试次数,Gitea 启动失败")
|
| 281 |
-
exit(1)
|
| 282 |
-
continue
|
| 283 |
-
|
| 284 |
-
# 监控 Gitea 输出
|
| 285 |
-
while True:
|
| 286 |
-
output = gitea_process.stdout.readline()
|
| 287 |
-
if output == '' and gitea_process.poll() is not None:
|
| 288 |
-
break
|
| 289 |
-
if output:
|
| 290 |
-
logger.info(f"Gitea: {output.strip()}")
|
| 291 |
-
|
| 292 |
-
return_code = gitea_process.poll()
|
| 293 |
-
if return_code != 0:
|
| 294 |
-
error_output = gitea_process.stderr.read()
|
| 295 |
-
logger.error(f"❌ Gitea 服务器异常退出,返回码: {return_code}")
|
| 296 |
-
logger.error(f"错误信息: {error_output}")
|
| 297 |
-
attempt += 1
|
| 298 |
-
if attempt < max_restart_attempts:
|
| 299 |
-
logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
|
| 300 |
-
time.sleep(restart_delay)
|
| 301 |
-
else:
|
| 302 |
-
logger.error(f"❌ 达到最大重试次数,Gitea 启动失败")
|
| 303 |
-
exit(1)
|
| 304 |
-
else:
|
| 305 |
-
logger.info("✅ Gitea 服务器正常退出")
|
| 306 |
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
logger.error(f"❌ 启动 Gitea 服务器时发生错误: {e}")
|
| 313 |
-
attempt += 1
|
| 314 |
-
if attempt < max_restart_attempts:
|
| 315 |
-
logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
|
| 316 |
-
time.sleep(restart_delay)
|
| 317 |
-
else:
|
| 318 |
-
logger.error(f"❌ 达到最大重试次数,Gitea 启动失败")
|
| 319 |
-
exit(1)
|
| 320 |
|
| 321 |
gitea_thread = threading.Thread(target=run_gitea)
|
| 322 |
gitea_thread.daemon = True
|
|
@@ -356,7 +321,7 @@ async def main():
|
|
| 356 |
|
| 357 |
logger.info("✅ 所有服务已启动完成!")
|
| 358 |
logger.info("📁 目录监控: /data → Hugging Face Hub")
|
| 359 |
-
logger.info(
|
| 360 |
logger.info("🛑 按 Ctrl+C 停止所有服务")
|
| 361 |
|
| 362 |
try:
|
|
@@ -367,7 +332,6 @@ async def main():
|
|
| 367 |
|
| 368 |
except Exception as e:
|
| 369 |
logger.error(f"❌ 启动服务时发生错误: {e}")
|
| 370 |
-
exit(1)
|
| 371 |
finally:
|
| 372 |
if 'observer' in locals():
|
| 373 |
observer.stop()
|
|
|
|
| 1 |
FROM gitea/gitea:1.24.6
|
| 2 |
|
| 3 |
+
# 以 root 用户设置权限(Spaces 允许构建时 root)
|
| 4 |
USER root
|
| 5 |
|
| 6 |
+
# 安装 Python、pip、git 和异步库
|
| 7 |
RUN apk update && \
|
| 8 |
+
apk add python3 py3-pip git && \
|
| 9 |
pip3 install --break-system-packages --upgrade pip
|
| 10 |
|
| 11 |
+
# 安装 Python 包,包括 pytz 用于处理时区
|
| 12 |
RUN pip3 install --break-system-packages watchdog huggingface_hub aiohttp pytz
|
| 13 |
|
| 14 |
# 预创建 Gitea 所需目录并设置权限
|
|
|
|
| 23 |
from huggingface_hub import snapshot_download
|
| 24 |
import logging
|
| 25 |
import base64
|
| 26 |
+
from datetime import datetime
|
| 27 |
+
import pytz
|
| 28 |
|
| 29 |
+
# 配置日志,使用北京时间
|
| 30 |
+
beijing_tz = pytz.timezone('Asia/Shanghai')
|
| 31 |
logging.basicConfig(
|
| 32 |
level=logging.INFO,
|
| 33 |
+
format='%(asctime)s - %(levelname)s - %(message)s',
|
| 34 |
+
datefmt='%Y-%m-%d %H:%M:%S %Z'
|
| 35 |
)
|
| 36 |
logger = logging.getLogger(__name__)
|
| 37 |
|
| 38 |
+
# 自定义日志格式化器以使用北京时间
|
| 39 |
+
class BeijingTimeFormatter(logging.Formatter):
|
| 40 |
+
def formatTime(self, record, datefmt=None):
|
| 41 |
+
dt = datetime.fromtimestamp(record.created, tz=beijing_tz)
|
| 42 |
+
if datefmt:
|
| 43 |
+
return dt.strftime(datefmt)
|
| 44 |
+
return dt.strftime("%Y-%m-%d %H:%M:%S %Z")
|
| 45 |
+
|
| 46 |
+
# 应用自定义格式化器
|
| 47 |
+
for handler in logging.getLogger().handlers:
|
| 48 |
+
handler.setFormatter(BeijingTimeFormatter(
|
| 49 |
+
fmt='%(asctime)s - %(levelname)s - %(message)s',
|
| 50 |
+
datefmt='%Y-%m-%d %H:%M:%S %Z'
|
| 51 |
+
))
|
| 52 |
+
|
| 53 |
def pull_from_hf_hub(repo_id, data_directory="/data"):
|
| 54 |
"""从 Hugging Face Hub 数据集仓库拉取数据替换 /data 目录"""
|
| 55 |
# 硬编码的 base64 编码的 HF_TOKEN
|
|
|
|
| 128 |
exit(1)
|
| 129 |
EOF
|
| 130 |
|
| 131 |
+
# 复制上传脚本,移除检测到事件日志,修改日志为北京时间
|
| 132 |
COPY --chown=git:git <<'EOF' /uploadhf.py
|
| 133 |
import os
|
| 134 |
import time
|
|
|
|
| 145 |
from datetime import datetime
|
| 146 |
import pytz
|
| 147 |
|
| 148 |
+
# 配置日志,使用北京时间
|
| 149 |
beijing_tz = pytz.timezone('Asia/Shanghai')
|
| 150 |
logging.basicConfig(
|
| 151 |
level=logging.INFO,
|
| 152 |
format='%(asctime)s - %(levelname)s - %(message)s',
|
| 153 |
+
datefmt='%Y-%m-%d %H:%M:%S %Z'
|
|
|
|
|
|
|
|
|
|
| 154 |
)
|
|
|
|
|
|
|
| 155 |
logger = logging.getLogger(__name__)
|
| 156 |
|
| 157 |
+
# 自定义日志格式化器以使用北京时间
|
| 158 |
+
class BeijingTimeFormatter(logging.Formatter):
|
| 159 |
+
def formatTime(self, record, datefmt=None):
|
| 160 |
+
dt = datetime.fromtimestamp(record.created, tz=beijing_tz)
|
| 161 |
+
if datefmt:
|
| 162 |
+
return dt.strftime(datefmt)
|
| 163 |
+
return dt.strftime("%Y-%m-%d %H:%M:%S %Z")
|
| 164 |
+
|
| 165 |
+
# 应用自定义格式化器
|
| 166 |
+
for handler in logging.getLogger().handlers:
|
| 167 |
+
handler.setFormatter(BeijingTimeFormatter(
|
| 168 |
+
fmt='%(asctime)s - %(levelname)s - %(message)s',
|
| 169 |
+
datefmt='%Y-%m-%d %H:%M:%S %Z'
|
| 170 |
+
))
|
| 171 |
+
|
| 172 |
class DataDirectoryHandler(FileSystemEventHandler):
|
| 173 |
"""处理 /data 目录文件变化的监控器"""
|
| 174 |
|
|
|
|
| 180 |
self.last_commit_time = 0
|
| 181 |
self.commit_delay = 1 # 防抖延迟 1 秒
|
| 182 |
self.pending_changes = [] # 缓冲待上传变更
|
|
|
|
| 183 |
logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
|
| 184 |
|
| 185 |
def on_any_event(self, event):
|
|
|
|
| 194 |
current_time = time.time()
|
| 195 |
if current_time - self.last_commit_time > self.commit_delay:
|
| 196 |
self.last_commit_time = current_time
|
| 197 |
+
asyncio.run(self.commit_changes(change_type))
|
|
|
|
| 198 |
|
| 199 |
async def commit_changes(self, change_type):
|
| 200 |
"""异步提交变更到 Hugging Face Hub,带重试机制"""
|
|
|
|
| 204 |
self.pending_changes = [] # 清空缓冲区
|
| 205 |
for attempt in range(max_retries):
|
| 206 |
try:
|
| 207 |
+
commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
|
| 208 |
logger.info(f"开始上传: {commit_message}")
|
| 209 |
await asyncio.to_thread(self.api.upload_folder,
|
| 210 |
folder_path=self.data_directory,
|
|
|
|
| 231 |
logger.info(f"HF_TOKEN value: {hf_token}")
|
| 232 |
|
| 233 |
if not repo_id:
|
| 234 |
+
raise ValueError("必须提供 repo_id 参数")
|
|
|
|
| 235 |
|
| 236 |
if not os.path.exists(data_directory):
|
| 237 |
logger.warning(f"监控目录 {data_directory} 不存在,正在创建...")
|
|
|
|
| 251 |
return observer
|
| 252 |
|
| 253 |
def start_gitea_server(port=7860):
|
| 254 |
+
"""启动 Gitea 服务器"""
|
| 255 |
def run_gitea():
|
| 256 |
+
try:
|
| 257 |
+
gitea_process = subprocess.Popen(
|
| 258 |
+
['gitea', 'web', '--port', str(port)],
|
| 259 |
+
stdout=subprocess.PIPE,
|
| 260 |
+
stderr=subprocess.PIPE,
|
| 261 |
+
universal_newlines=True
|
| 262 |
+
)
|
| 263 |
+
logger.info(f"🚀 Gitea 服务器已启动,端口: {port}")
|
| 264 |
+
logger.info(f"📊 访问地址: http://localhost:{port}")
|
| 265 |
+
|
| 266 |
+
while True:
|
| 267 |
+
output = gitea_process.stdout.readline()
|
| 268 |
+
if output == '' and gitea_process.poll() is not None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
break
|
| 270 |
+
if output:
|
| 271 |
+
logger.info(f"Gitea: {output.strip()}")
|
| 272 |
+
|
| 273 |
+
return_code = gitea_process.poll()
|
| 274 |
+
if return_code != 0:
|
| 275 |
+
error_output = gitea_process.stderr.read()
|
| 276 |
+
logger.error(f"❌ Gitea 服务器异常退出,返回码: {return_code}")
|
| 277 |
+
logger.error(f"错误信息: {error_output}")
|
| 278 |
+
else:
|
| 279 |
+
logger.info("✅ Gitea 服务器正常退出")
|
| 280 |
|
| 281 |
+
except FileNotFoundError:
|
| 282 |
+
logger.error("❌ 未找到 gitea 命令,请确保 Gitea 已正确安装")
|
| 283 |
+
except Exception as e:
|
| 284 |
+
logger.error(f"❌ 启动 Gitea 服务器时发生错误: {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
|
| 286 |
gitea_thread = threading.Thread(target=run_gitea)
|
| 287 |
gitea_thread.daemon = True
|
|
|
|
| 321 |
|
| 322 |
logger.info("✅ 所有服务已启动完成!")
|
| 323 |
logger.info("📁 目录监控: /data → Hugging Face Hub")
|
| 324 |
+
logger.info("🌐 Gitea 服务: http://localhost:7860")
|
| 325 |
logger.info("🛑 按 Ctrl+C 停止所有服务")
|
| 326 |
|
| 327 |
try:
|
|
|
|
| 332 |
|
| 333 |
except Exception as e:
|
| 334 |
logger.error(f"❌ 启动服务时发生错误: {e}")
|
|
|
|
| 335 |
finally:
|
| 336 |
if 'observer' in locals():
|
| 337 |
observer.stop()
|