sync_hub.py

import os
from pathlib import Path

HF_TOKEN = os.environ.get("HF_TOKEN", "")
DATASET_REPO = os.environ.get("OPENCLAW_DATASET_REPO", "")
PERSISTENT_FILES = {"topic_history.json", "fact_history.json", "token.pickle"}

_api = None
def _get_api():
    global _api
    if _api is None:
        from huggingface_hub import HfApi
        _api = HfApi(token=HF_TOKEN)
    return _api

def pull_state(base_dir: Path):
    if not DATASET_REPO or not HF_TOKEN:
        print("OPENCLAW_DATASET_REPO or HF_TOKEN not set, skipping pull"); return
    from huggingface_hub import hf_hub_download
    from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
    print(f"Pulling state from {DATASET_REPO}...")
    for fname in PERSISTENT_FILES:
        try:
            hf_hub_download(repo_id=DATASET_REPO, filename=fname, repo_type="dataset",
                            token=HF_TOKEN, local_dir=str(base_dir), local_dir_use_symlinks=False)
            print(f"  pulled {fname}")
        except (EntryNotFoundError, RepositoryNotFoundError): pass
        except Exception as e: print(f"  could not pull {fname}: {e}")

def push_file(local_path: Path):
    if not DATASET_REPO or not HF_TOKEN: return
    if local_path.name not in PERSISTENT_FILES or not local_path.exists(): return
    try:
        _get_api().upload_file(
            path_or_fileobj=str(local_path), path_in_repo=local_path.name,
            repo_id=DATASET_REPO, repo_type="dataset", token=HF_TOKEN,
            commit_message=f"auto: {local_path.name}")
        print(f"  synced {local_path.name} to Hub")
    except Exception as e:
        print(f"  Hub sync failed for {local_path.name}: {e}")

def push_all_state(base_dir: Path):
    for fname in PERSISTENT_FILES:
        p = base_dir / fname
        if p.exists(): push_file(p)