整理中文 model card 與 SSD streaming 定位

977a65f 27 days ago

4.92 kB

#!/usr/bin/env python3
"""
Auto-upload system for Hugging Face Spaces.
- Runs validation and pushes on success
- Watches for file changes and auto-commits/pushes
- Can resume from repo after crash
"""

import os
import sys
import subprocess
import time
import signal
import threading
from pathlib import Path
from datetime import datetime

HF_TOKEN = os.environ.get("HF_TOKEN", "")
# ★ 這三個值不可寫死: 寫死會把 4B 的成果推到 27B 專案 / 別的 Space, 等於成果憑空消失
#   (本檔是從 sddqwen38-27b 沿用的, 那時這裡是 27B 的值)。
#   用 REPO_DIR 推導 repo 名: .../sddqwen35-4b -> HelloSun/sddqwen35-4b
STORAGE_REPO = os.environ.get("STORAGE_REPO", "HelloSun/qwen35-4b-ssd-streaming")
SPACE_REPO = os.environ.get("SPACE_REPO", "")
REPO_DIR = Path(os.environ.get("REPO_DIR", "/app"))
SPACE_URL = f"https://huggingface.co/spaces/{SPACE_REPO}" if SPACE_REPO else ""
MODEL_REPO_URL = f"https://huggingface.co/{STORAGE_REPO}"

def run_cmd(cmd, cwd=None, capture=True, timeout=300):
    """Run command and return (success, stdout, stderr)."""
    try:
        result = subprocess.run(
            cmd, shell=True, cwd=cwd or REPO_DIR,
            capture_output=capture, text=True, timeout=timeout
        )
        return result.returncode == 0, result.stdout, result.stderr
    except subprocess.TimeoutExpired:
        return False, "", "Timeout"
    except Exception as e:
        return False, "", str(e)

def git_setup():
    """Configure git with HF token."""
    run_cmd(f'git config user.email "agent@huggingface.co"')
    run_cmd(f'git config user.name "Auto Upload Agent"')
    # ★ 目標 repo 由 STORAGE_REPO 決定 (預設本專案), 不要再寫死 27B 的 Space 名。
    if HF_TOKEN:
        target = f"spaces/{SPACE_REPO}" if SPACE_REPO else STORAGE_REPO
        run_cmd(f'git remote set-url origin https://oauth2:{HF_TOKEN}@huggingface.co/{target}')

def run_validation():
    """Run installpi.sh validation and return success."""
    print("[AUTO] Running validation...")
    success, out, err = run_cmd("./installpi.sh", timeout=600)
    print(out[-2000:] if out else "")
    if err:
        print(err[-2000:], file=sys.stderr)
    return success

def commit_and_push(message=None):
    """Commit all changes and push to HF."""
    if message is None:
        message = f"Auto-update: {datetime.now().isoformat(timespec='seconds')}"
    
    # Add all files
    success, _, _ = run_cmd("git add -A")
    if not success:
        print("[AUTO] git add failed")
        return False
    
    # Check if there are changes
    success, stdout, _ = run_cmd("git diff --cached --quiet")
    if success:
        print("[AUTO] No changes to commit")
        return True
    
    # Commit
    success, _, err = run_cmd(f'git commit -m "{message}"')
    if not success:
        print(f"[AUTO] Commit failed: {err}")
        return False
    
    # Push
    success, _, err = run_cmd("git push origin main")
    if not success:
        print(f"[AUTO] Push failed: {err}")
        return False
    
    print(f"[AUTO] Pushed: {message}")
    return True

def watch_and_upload():
    """Watch for file changes and auto-upload."""
    print("[AUTO] Starting file watcher...")
    last_hash = ""
    
    def get_git_hash():
        success, stdout, _ = run_cmd("git rev-parse HEAD")
        return stdout.strip() if success else ""
    
    last_hash = get_git_hash()
    
    while True:
        time.sleep(10)
        current_hash = get_git_hash()
        if current_hash != last_hash:
            print("[AUTO] Detected repo change, uploading...")
            commit_and_push("Auto-sync after code change")
            last_hash = current_hash

def main():
    print("=" * 60)
    print("AUTO UPLOAD AGENT")
    print("=" * 60)
    
    git_setup()
    
    # Check if we're resuming from a crash
    success, stdout, _ = run_cmd("git status --porcelain")
    if stdout.strip():
        print("[AUTO] Uncommitted changes detected, committing...")
        commit_and_push("Recovery: uncommitted changes from crash")
    
    # Run validation first
    if run_validation():
        print("[AUTO] Validation passed! Pushing first version...")
        commit_and_push("First version: validation passed (4K context)")
    else:
        print("[AUTO] Validation failed, not pushing")
        return 1
    
    # Start watcher in background
    watcher_thread = threading.Thread(target=watch_and_upload, daemon=True)
    watcher_thread.start()
    
    # Keep running
    print("[AUTO] Running... Press Ctrl+C to stop")
    try:
        while True:
            time.sleep(60)
            # Periodic sync in case watcher missed something
            commit_and_push("Periodic sync")
    except KeyboardInterrupt:
        print("\n[AUTO] Stopping...")
        commit_and_push("Final sync before shutdown")
    
    return 0

if __name__ == "__main__":
    sys.exit(main())

	#!/usr/bin/env python3
	"""
	Auto-upload system for Hugging Face Spaces.
	- Runs validation and pushes on success
	- Watches for file changes and auto-commits/pushes
	- Can resume from repo after crash
	"""

	import os
	import sys
	import subprocess
	import time
	import signal
	import threading
	from pathlib import Path
	from datetime import datetime

	HF_TOKEN = os.environ.get("HF_TOKEN", "")
	# ★ 這三個值不可寫死: 寫死會把 4B 的成果推到 27B 專案 / 別的 Space, 等於成果憑空消失
	# (本檔是從 sddqwen38-27b 沿用的, 那時這裡是 27B 的值)。
	# 用 REPO_DIR 推導 repo 名: .../sddqwen35-4b -> HelloSun/sddqwen35-4b
	STORAGE_REPO = os.environ.get("STORAGE_REPO", "HelloSun/qwen35-4b-ssd-streaming")
	SPACE_REPO = os.environ.get("SPACE_REPO", "")
	REPO_DIR = Path(os.environ.get("REPO_DIR", "/app"))
	SPACE_URL = f"https://huggingface.co/spaces/{SPACE_REPO}" if SPACE_REPO else ""
	MODEL_REPO_URL = f"https://huggingface.co/{STORAGE_REPO}"

	def run_cmd(cmd, cwd=None, capture=True, timeout=300):
	"""Run command and return (success, stdout, stderr)."""
	try:
	result = subprocess.run(
	cmd, shell=True, cwd=cwd or REPO_DIR,
	capture_output=capture, text=True, timeout=timeout
	)
	return result.returncode == 0, result.stdout, result.stderr
	except subprocess.TimeoutExpired:
	return False, "", "Timeout"
	except Exception as e:
	return False, "", str(e)

	def git_setup():
	"""Configure git with HF token."""
	run_cmd(f'git config user.email "agent@huggingface.co"')
	run_cmd(f'git config user.name "Auto Upload Agent"')
	# ★ 目標 repo 由 STORAGE_REPO 決定 (預設本專案), 不要再寫死 27B 的 Space 名。
	if HF_TOKEN:
	target = f"spaces/{SPACE_REPO}" if SPACE_REPO else STORAGE_REPO
	run_cmd(f'git remote set-url origin https://oauth2:{HF_TOKEN}@huggingface.co/{target}')

	def run_validation():
	"""Run installpi.sh validation and return success."""
	print("[AUTO] Running validation...")
	success, out, err = run_cmd("./installpi.sh", timeout=600)
	print(out[-2000:] if out else "")
	if err:
	print(err[-2000:], file=sys.stderr)
	return success

	def commit_and_push(message=None):
	"""Commit all changes and push to HF."""
	if message is None:
	message = f"Auto-update: {datetime.now().isoformat(timespec='seconds')}"

	# Add all files
	success, _, _ = run_cmd("git add -A")
	if not success:
	print("[AUTO] git add failed")
	return False

	# Check if there are changes
	success, stdout, _ = run_cmd("git diff --cached --quiet")
	if success:
	print("[AUTO] No changes to commit")
	return True

	# Commit
	success, _, err = run_cmd(f'git commit -m "{message}"')
	if not success:
	print(f"[AUTO] Commit failed: {err}")
	return False

	# Push
	success, _, err = run_cmd("git push origin main")
	if not success:
	print(f"[AUTO] Push failed: {err}")
	return False

	print(f"[AUTO] Pushed: {message}")
	return True

	def watch_and_upload():
	"""Watch for file changes and auto-upload."""
	print("[AUTO] Starting file watcher...")
	last_hash = ""

	def get_git_hash():
	success, stdout, _ = run_cmd("git rev-parse HEAD")
	return stdout.strip() if success else ""

	last_hash = get_git_hash()

	while True:
	time.sleep(10)
	current_hash = get_git_hash()
	if current_hash != last_hash:
	print("[AUTO] Detected repo change, uploading...")
	commit_and_push("Auto-sync after code change")
	last_hash = current_hash

	def main():
	print("=" * 60)
	print("AUTO UPLOAD AGENT")
	print("=" * 60)

	git_setup()

	# Check if we're resuming from a crash
	success, stdout, _ = run_cmd("git status --porcelain")
	if stdout.strip():
	print("[AUTO] Uncommitted changes detected, committing...")
	commit_and_push("Recovery: uncommitted changes from crash")

	# Run validation first
	if run_validation():
	print("[AUTO] Validation passed! Pushing first version...")
	commit_and_push("First version: validation passed (4K context)")
	else:
	print("[AUTO] Validation failed, not pushing")
	return 1

	# Start watcher in background
	watcher_thread = threading.Thread(target=watch_and_upload, daemon=True)
	watcher_thread.start()

	# Keep running
	print("[AUTO] Running... Press Ctrl+C to stop")
	try:
	while True:
	time.sleep(60)
	# Periodic sync in case watcher missed something
	commit_and_push("Periodic sync")
	except KeyboardInterrupt:
	print("\n[AUTO] Stopping...")
	commit_and_push("Final sync before shutdown")

	return 0

	if __name__ == "__main__":
	sys.exit(main())