Update Dockerfile
Browse files- Dockerfile +28 -25
Dockerfile
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
FROM gitea/gitea:1.24.6
|
| 2 |
|
| 3 |
# 以 root 用户设置权限(Spaces 允许构建时 root)
|
| 4 |
USER root
|
|
@@ -114,21 +114,21 @@ def pull_from_hf_hub(repo_id, data_directory="/data"):
|
|
| 114 |
os.system("ls -la /data/gitea/conf 2>/dev/null || echo '/data/gitea/conf 不存在'")
|
| 115 |
os.system("ls -la /data/gitea/log 2>/dev/null || echo '/data/gitea/log 不存在'")
|
| 116 |
|
| 117 |
-
logger.info(f"
|
| 118 |
return True
|
| 119 |
|
| 120 |
except Exception as e:
|
| 121 |
-
logger.error(f"
|
| 122 |
return False
|
| 123 |
|
| 124 |
if __name__ == "__main__":
|
| 125 |
repo_id = os.getenv("REPO_ID", "02engine/02gitea")
|
| 126 |
if not pull_from_hf_hub(repo_id):
|
| 127 |
-
logger.error("
|
| 128 |
exit(1)
|
| 129 |
EOF
|
| 130 |
|
| 131 |
-
# 复制上传脚本
|
| 132 |
COPY --chown=git:git <<'EOF' /uploadhf.py
|
| 133 |
import os
|
| 134 |
import time
|
|
@@ -141,6 +141,7 @@ from pathlib import Path
|
|
| 141 |
from watchdog.observers import Observer
|
| 142 |
from watchdog.events import FileSystemEventHandler
|
| 143 |
from huggingface_hub import HfApi
|
|
|
|
| 144 |
import base64
|
| 145 |
from datetime import datetime
|
| 146 |
import pytz
|
|
@@ -206,21 +207,23 @@ class DataDirectoryHandler(FileSystemEventHandler):
|
|
| 206 |
try:
|
| 207 |
commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
|
| 208 |
logger.info(f"开始上传: {commit_message}")
|
| 209 |
-
await asyncio.to_thread(
|
|
|
|
| 210 |
folder_path=self.data_directory,
|
| 211 |
repo_id=self.repo_id,
|
| 212 |
repo_type="dataset",
|
| 213 |
commit_message=commit_message,
|
| 214 |
-
ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
|
|
|
|
| 215 |
)
|
| 216 |
-
logger.info(f"
|
| 217 |
return
|
| 218 |
except Exception as e:
|
| 219 |
-
logger.error(f"
|
| 220 |
if attempt < max_retries - 1:
|
| 221 |
logger.info(f"将在 {retry_delay} 秒后重试...")
|
| 222 |
await asyncio.sleep(retry_delay)
|
| 223 |
-
logger.error(f"
|
| 224 |
|
| 225 |
def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
|
| 226 |
"""启动目录监控服务"""
|
|
@@ -246,7 +249,7 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
|
|
| 246 |
observer = Observer()
|
| 247 |
observer.schedule(event_handler, data_directory, recursive=True)
|
| 248 |
observer.start()
|
| 249 |
-
logger.info(f"
|
| 250 |
|
| 251 |
return observer
|
| 252 |
|
|
@@ -260,8 +263,8 @@ def start_gitea_server(port=7860):
|
|
| 260 |
stderr=subprocess.PIPE,
|
| 261 |
universal_newlines=True
|
| 262 |
)
|
| 263 |
-
logger.info(f"
|
| 264 |
-
logger.info(f"
|
| 265 |
|
| 266 |
while True:
|
| 267 |
output = gitea_process.stdout.readline()
|
|
@@ -273,15 +276,15 @@ def start_gitea_server(port=7860):
|
|
| 273 |
return_code = gitea_process.poll()
|
| 274 |
if return_code != 0:
|
| 275 |
error_output = gitea_process.stderr.read()
|
| 276 |
-
logger.error(f"
|
| 277 |
logger.error(f"错误信息: {error_output}")
|
| 278 |
else:
|
| 279 |
-
logger.info("
|
| 280 |
|
| 281 |
except FileNotFoundError:
|
| 282 |
-
logger.error("
|
| 283 |
except Exception as e:
|
| 284 |
-
logger.error(f"
|
| 285 |
|
| 286 |
gitea_thread = threading.Thread(target=run_gitea)
|
| 287 |
gitea_thread.daemon = True
|
|
@@ -304,13 +307,13 @@ async def main():
|
|
| 304 |
|
| 305 |
while retry_count < max_retries:
|
| 306 |
try:
|
| 307 |
-
logger.info(f"
|
| 308 |
|
| 309 |
# 先运行 pullhf.py 拉取最新数据集
|
| 310 |
logger.info("运行 pullhf.py 拉取最新数据集")
|
| 311 |
pull_result = subprocess.run(["python3", "/pullhf.py"], check=True)
|
| 312 |
if pull_result.returncode != 0:
|
| 313 |
-
logger.error("
|
| 314 |
exit(1)
|
| 315 |
|
| 316 |
# 启动 Gitea 服务器
|
|
@@ -323,10 +326,10 @@ async def main():
|
|
| 323 |
hf_token=CONFIG["hf_token"]
|
| 324 |
)
|
| 325 |
|
| 326 |
-
logger.info("
|
| 327 |
-
logger.info("
|
| 328 |
-
logger.info("
|
| 329 |
-
logger.info("
|
| 330 |
|
| 331 |
try:
|
| 332 |
while True:
|
|
@@ -337,12 +340,12 @@ async def main():
|
|
| 337 |
|
| 338 |
except Exception as e:
|
| 339 |
retry_count += 1
|
| 340 |
-
logger.error(f"
|
| 341 |
if retry_count < max_retries:
|
| 342 |
logger.info(f"将在 5 秒后尝试重启...")
|
| 343 |
await asyncio.sleep(5)
|
| 344 |
else:
|
| 345 |
-
logger.error(f"
|
| 346 |
exit(1)
|
| 347 |
finally:
|
| 348 |
if 'observer' in locals():
|
|
|
|
| 1 |
+
FROM gitea/gitea:1.24.6
|
| 2 |
|
| 3 |
# 以 root 用户设置权限(Spaces 允许构建时 root)
|
| 4 |
USER root
|
|
|
|
| 114 |
os.system("ls -la /data/gitea/conf 2>/dev/null || echo '/data/gitea/conf 不存在'")
|
| 115 |
os.system("ls -la /data/gitea/log 2>/dev/null || echo '/data/gitea/log 不存在'")
|
| 116 |
|
| 117 |
+
logger.info(f"成功从 Hugging Face Hub 拉取数据到 {data_directory}")
|
| 118 |
return True
|
| 119 |
|
| 120 |
except Exception as e:
|
| 121 |
+
logger.error(f"拉取 Hugging Face 数据集失败: {e}")
|
| 122 |
return False
|
| 123 |
|
| 124 |
if __name__ == "__main__":
|
| 125 |
repo_id = os.getenv("REPO_ID", "02engine/02gitea")
|
| 126 |
if not pull_from_hf_hub(repo_id):
|
| 127 |
+
logger.error("拉取数据集失败,退出")
|
| 128 |
exit(1)
|
| 129 |
EOF
|
| 130 |
|
| 131 |
+
# 复制上传脚本(已禁用进度条)
|
| 132 |
COPY --chown=git:git <<'EOF' /uploadhf.py
|
| 133 |
import os
|
| 134 |
import time
|
|
|
|
| 141 |
from watchdog.observers import Observer
|
| 142 |
from watchdog.events import FileSystemEventHandler
|
| 143 |
from huggingface_hub import HfApi
|
| 144 |
+
from huggingface_hub.utils import tqdm_utils # 用于禁用进度条
|
| 145 |
import base64
|
| 146 |
from datetime import datetime
|
| 147 |
import pytz
|
|
|
|
| 207 |
try:
|
| 208 |
commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
|
| 209 |
logger.info(f"开始上传: {commit_message}")
|
| 210 |
+
await asyncio.to_thread(
|
| 211 |
+
self.api.upload_folder,
|
| 212 |
folder_path=self.data_directory,
|
| 213 |
repo_id=self.repo_id,
|
| 214 |
repo_type="dataset",
|
| 215 |
commit_message=commit_message,
|
| 216 |
+
ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"],
|
| 217 |
+
tqdm_class=tqdm_utils.DisableTqdm # 禁用进度条
|
| 218 |
)
|
| 219 |
+
logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
|
| 220 |
return
|
| 221 |
except Exception as e:
|
| 222 |
+
logger.error(f"提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
|
| 223 |
if attempt < max_retries - 1:
|
| 224 |
logger.info(f"将在 {retry_delay} 秒后重试...")
|
| 225 |
await asyncio.sleep(retry_delay)
|
| 226 |
+
logger.error(f"达到最大重试次数,上传失败")
|
| 227 |
|
| 228 |
def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
|
| 229 |
"""启动目录监控服务"""
|
|
|
|
| 249 |
observer = Observer()
|
| 250 |
observer.schedule(event_handler, data_directory, recursive=True)
|
| 251 |
observer.start()
|
| 252 |
+
logger.info(f"目录监控服务已启动: {data_directory}")
|
| 253 |
|
| 254 |
return observer
|
| 255 |
|
|
|
|
| 263 |
stderr=subprocess.PIPE,
|
| 264 |
universal_newlines=True
|
| 265 |
)
|
| 266 |
+
logger.info(f"Gitea 服务器已启动,端口: {port}")
|
| 267 |
+
logger.info(f"访问地址: http://localhost:{port}")
|
| 268 |
|
| 269 |
while True:
|
| 270 |
output = gitea_process.stdout.readline()
|
|
|
|
| 276 |
return_code = gitea_process.poll()
|
| 277 |
if return_code != 0:
|
| 278 |
error_output = gitea_process.stderr.read()
|
| 279 |
+
logger.error(f"Gitea 服务器异常退出,返回码: {return_code}")
|
| 280 |
logger.error(f"错误信息: {error_output}")
|
| 281 |
else:
|
| 282 |
+
logger.info("Gitea 服务器正常退出")
|
| 283 |
|
| 284 |
except FileNotFoundError:
|
| 285 |
+
logger.error("未找到 gitea 命令,请确保 Gitea 已正确安装")
|
| 286 |
except Exception as e:
|
| 287 |
+
logger.error(f"启动 Gitea 服务器时发生错误: {e}")
|
| 288 |
|
| 289 |
gitea_thread = threading.Thread(target=run_gitea)
|
| 290 |
gitea_thread.daemon = True
|
|
|
|
| 307 |
|
| 308 |
while retry_count < max_retries:
|
| 309 |
try:
|
| 310 |
+
logger.info(f"启动集成服务 (尝试 {retry_count + 1}/{max_retries})...")
|
| 311 |
|
| 312 |
# 先运行 pullhf.py 拉取最新数据集
|
| 313 |
logger.info("运行 pullhf.py 拉取最新数据集")
|
| 314 |
pull_result = subprocess.run(["python3", "/pullhf.py"], check=True)
|
| 315 |
if pull_result.returncode != 0:
|
| 316 |
+
logger.error("pullhf.py 执行失败,退出")
|
| 317 |
exit(1)
|
| 318 |
|
| 319 |
# 启动 Gitea 服务器
|
|
|
|
| 326 |
hf_token=CONFIG["hf_token"]
|
| 327 |
)
|
| 328 |
|
| 329 |
+
logger.info("所有服务已启动完成!")
|
| 330 |
+
logger.info("目录监控: /data to Hugging Face Hub")
|
| 331 |
+
logger.info("Gitea 服务: http://localhost:7860")
|
| 332 |
+
logger.info("按 Ctrl+C 停止所有服务")
|
| 333 |
|
| 334 |
try:
|
| 335 |
while True:
|
|
|
|
| 340 |
|
| 341 |
except Exception as e:
|
| 342 |
retry_count += 1
|
| 343 |
+
logger.error(f"服务崩溃 (尝试 {retry_count}/{max_retries}): {e}")
|
| 344 |
if retry_count < max_retries:
|
| 345 |
logger.info(f"将在 5 秒后尝试重启...")
|
| 346 |
await asyncio.sleep(5)
|
| 347 |
else:
|
| 348 |
+
logger.error(f"达到最大重试次数 ({max_retries}),退出")
|
| 349 |
exit(1)
|
| 350 |
finally:
|
| 351 |
if 'observer' in locals():
|