workspace/scripts/auto_sync_hf.py

#!/usr/bin/env python3
"""
Auto-sync script: Monitors changes and pushes to Hugging Face realtime.
Runs as daemon, checks every 5 minutes for changes.
"""
import time
import subprocess
import sys
from pathlib import Path
from huggingface_hub import HfApi, upload_folder
from datetime import datetime

REPO_ID = "anhtld/vla"
CHECK_INTERVAL = 300  # 5 minutes
REPO_ROOT = Path("/lustre09/project/6037638/knguy52/vla")

# Files/dirs to ignore (same as .gitignore)
IGNORE_PATTERNS = [
    ".git/*",
    ".venv/*",
    # Don't ignore checkpoints - we want them synced!
    # "*.pt",
    # "*.pth",
    # "*.ckpt",
    "*.h5",
    "*.hdf5",
    "*.pkl",
    "*.pickle",
    # Keep logs for monitoring
    # "logs/*",
    "*.out",
    "*.err",
    "slurm-*.out",
    "__pycache__/*",
    ".pytest_cache/*",
    ".ruff_cache/*",
    # Don't ignore outputs - want results synced
    # "outputs/*",
    "wandb/*",
    "scratch/*",
    "*token*",
    "*secret*",
    ".env*",
]

def get_last_commit_time():
    """Get timestamp of last git commit"""
    try:
        result = subprocess.run(
            ["git", "log", "-1", "--format=%ct"],
            cwd=REPO_ROOT,
            capture_output=True,
            text=True,
            check=True
        )
        return int(result.stdout.strip())
    except:
        return 0

def has_changes():
    """Check if there are uncommitted changes"""
    try:
        result = subprocess.run(
            ["git", "status", "--porcelain"],
            cwd=REPO_ROOT,
            capture_output=True,
            text=True,
            check=True
        )
        return bool(result.stdout.strip())
    except:
        return False

def sync_to_hf():
    """Sync current state to Hugging Face"""
    try:
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

        print(f"[{timestamp}] Syncing to Hugging Face...")

        # Upload via HF API (bypasses git auth issues)
        upload_folder(
            folder_path=str(REPO_ROOT),
            repo_id=REPO_ID,
            repo_type="model",
            ignore_patterns=IGNORE_PATTERNS,
            commit_message=f"Auto-sync: {timestamp}",
        )

        print(f"[{timestamp}] ✅ Sync complete")
        return True
    except Exception as e:
        print(f"[{timestamp}] ❌ Sync failed: {e}")
        return False

def main():
    print("="*60)
    print("🔄 DoVLA Auto-Sync Daemon Started")
    print(f"Repo: {REPO_ID}")
    print(f"Check interval: {CHECK_INTERVAL}s ({CHECK_INTERVAL//60} minutes)")
    print("="*60)
    print()

    last_sync_time = get_last_commit_time()

    while True:
        try:
            if has_changes():
                timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                print(f"[{timestamp}] Changes detected, syncing...")
                if sync_to_hf():
                    last_sync_time = time.time()
            else:
                timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                print(f"[{timestamp}] No changes, sleeping...")

            time.sleep(CHECK_INTERVAL)

        except KeyboardInterrupt:
            print("\n🛑 Auto-sync daemon stopped by user")
            sys.exit(0)
        except Exception as e:
            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            print(f"[{timestamp}] Error in main loop: {e}")
            time.sleep(CHECK_INTERVAL)

if __name__ == "__main__":
    main()