File size: 3,426 Bytes
68442cd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | #!/usr/bin/env python3
"""
Auto-sync script: Monitors changes and pushes to Hugging Face realtime.
Runs as daemon, checks every 5 minutes for changes.
"""
import time
import subprocess
import sys
from pathlib import Path
from huggingface_hub import HfApi, upload_folder
from datetime import datetime
REPO_ID = "anhtld/vla"
CHECK_INTERVAL = 300 # 5 minutes
REPO_ROOT = Path("/lustre09/project/6037638/knguy52/vla")
# Files/dirs to ignore (same as .gitignore)
IGNORE_PATTERNS = [
".git/*",
".venv/*",
# Don't ignore checkpoints - we want them synced!
# "*.pt",
# "*.pth",
# "*.ckpt",
"*.h5",
"*.hdf5",
"*.pkl",
"*.pickle",
# Keep logs for monitoring
# "logs/*",
"*.out",
"*.err",
"slurm-*.out",
"__pycache__/*",
".pytest_cache/*",
".ruff_cache/*",
# Don't ignore outputs - want results synced
# "outputs/*",
"wandb/*",
"scratch/*",
"*token*",
"*secret*",
".env*",
]
def get_last_commit_time():
"""Get timestamp of last git commit"""
try:
result = subprocess.run(
["git", "log", "-1", "--format=%ct"],
cwd=REPO_ROOT,
capture_output=True,
text=True,
check=True
)
return int(result.stdout.strip())
except:
return 0
def has_changes():
"""Check if there are uncommitted changes"""
try:
result = subprocess.run(
["git", "status", "--porcelain"],
cwd=REPO_ROOT,
capture_output=True,
text=True,
check=True
)
return bool(result.stdout.strip())
except:
return False
def sync_to_hf():
"""Sync current state to Hugging Face"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{timestamp}] Syncing to Hugging Face...")
# Upload via HF API (bypasses git auth issues)
upload_folder(
folder_path=str(REPO_ROOT),
repo_id=REPO_ID,
repo_type="model",
ignore_patterns=IGNORE_PATTERNS,
commit_message=f"Auto-sync: {timestamp}",
)
print(f"[{timestamp}] ✅ Sync complete")
return True
except Exception as e:
print(f"[{timestamp}] ❌ Sync failed: {e}")
return False
def main():
print("="*60)
print("🔄 DoVLA Auto-Sync Daemon Started")
print(f"Repo: {REPO_ID}")
print(f"Check interval: {CHECK_INTERVAL}s ({CHECK_INTERVAL//60} minutes)")
print("="*60)
print()
last_sync_time = get_last_commit_time()
while True:
try:
if has_changes():
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{timestamp}] Changes detected, syncing...")
if sync_to_hf():
last_sync_time = time.time()
else:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{timestamp}] No changes, sleeping...")
time.sleep(CHECK_INTERVAL)
except KeyboardInterrupt:
print("\n🛑 Auto-sync daemon stopped by user")
sys.exit(0)
except Exception as e:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{timestamp}] Error in main loop: {e}")
time.sleep(CHECK_INTERVAL)
if __name__ == "__main__":
main()
|