fix: use openclaw_data/ path, sanitize API keys before upload, add staging dir and tracebacks

scripts/sync_hf.py

@@ -7,8 +7,8 @@ Simplified persistence: upload/download the entire ~/.openclaw directory
 as-is to/from a Hugging Face Dataset repo.
 
 - Startup:  snapshot_download  →  ~/.openclaw
-- Periodic: upload_folder      →  dataset .openclaw/
-- Shutdown: final upload_folder →  dataset .openclaw/
+- Periodic: upload_folder      →  dataset openclaw_data/
+- Shutdown: final upload_folder →  dataset openclaw_data/
 """
 
 import os
@@ -18,6 +18,10 @@ import threading
 import subprocess
 import signal
 import json
+import shutil
+import tempfile
+import traceback
+import re
 from pathlib import Path
 from datetime import datetime
 from huggingface_hub import HfApi, snapshot_download
@@ -46,6 +50,9 @@ HF_TOKEN   = os.environ.get("HF_TOKEN")
 OPENCLAW_HOME = Path.home() / ".openclaw"
 APP_DIR       = Path("/app/openclaw")
 
+# Use "openclaw_data" (no dot prefix) to be visible in HF dataset browser
+DATASET_PATH = "openclaw_data"
+
 TELEGRAM_BOT_TOKEN  = os.environ.get("TELEGRAM_BOT_TOKEN", "")
 TELEGRAM_BOT_NAME   = os.environ.get("TELEGRAM_BOT_NAME", "opentauronbot")
 TELEGRAM_ALLOW_USER = os.environ.get("TELEGRAM_ALLOW_USER", "taoshen1")
@@ -75,6 +82,7 @@ EXCLUDE_PATTERNS = [
     "**/extensions/**",
     "**/.persistence.lock",
     "**/.persistence-state.json",
+    "**/sync.log",            # don't upload our own log
 ]
 
 # ── Sync Manager ────────────────────────────────────────────────────────────
@@ -104,49 +112,46 @@ class OpenClawFullSync:
     # ── Restore (startup) ─────────────────────────────────────────────
 
     def load_from_repo(self):
-        """Download the entire .openclaw/ directory from dataset → ~/.openclaw"""
+        """Download from dataset → ~/.openclaw"""
         if not self.enabled:
             return
         print(f"[SYNC] ▶ Restoring ~/.openclaw from dataset {HF_REPO_ID} ...")
         OPENCLAW_HOME.mkdir(parents=True, exist_ok=True)
 
         try:
-            # Check if .openclaw/ folder exists in dataset
             files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")
-            openclaw_files = [f for f in files if f.startswith(".openclaw/")]
+            openclaw_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]
             if not openclaw_files:
-                print("[SYNC] No .openclaw/ folder in dataset. Starting fresh.")
+                print(f"[SYNC] No {DATASET_PATH}/ folder in dataset. Starting fresh.")
                 self._ensure_default_config()
                 self._ensure_telegram_credentials()
                 return
 
-            print(f"[SYNC] Found {len(openclaw_files)} files under .openclaw/ in dataset")
+            print(f"[SYNC] Found {len(openclaw_files)} files under {DATASET_PATH}/ in dataset")
 
-            # Download .openclaw/* to a temp root, then move into place
-            import tempfile, shutil
             with tempfile.TemporaryDirectory() as tmpdir:
                 snapshot_download(
                     repo_id=HF_REPO_ID,
                     repo_type="dataset",
-                    allow_patterns=".openclaw/**",
+                    allow_patterns=f"{DATASET_PATH}/**",
                     local_dir=tmpdir,
                     token=HF_TOKEN,
                 )
-                downloaded_root = Path(tmpdir) / ".openclaw"
+                downloaded_root = Path(tmpdir) / DATASET_PATH
                 if downloaded_root.exists():
-                    # Copy all contents into ~/.openclaw, preserving existing
                     for item in downloaded_root.rglob("*"):
                         if item.is_file():
                             rel = item.relative_to(downloaded_root)
                             dest = OPENCLAW_HOME / rel
                             dest.parent.mkdir(parents=True, exist_ok=True)
                             shutil.copy2(str(item), str(dest))
-                    print(f"[SYNC] ✓ Restore completed.")
+                    print("[SYNC] ✓ Restore completed.")
                 else:
-                    print("[SYNC] Downloaded snapshot but .openclaw dir not found. Starting fresh.")
+                    print("[SYNC] Downloaded snapshot but dir not found. Starting fresh.")
 
         except Exception as e:
             print(f"[SYNC] ✗ Restore failed: {e}")
+            traceback.print_exc()
 
         # Patch config & telegram after restore
         self._patch_config()
@@ -156,40 +161,100 @@ class OpenClawFullSync:
     # ── Save (periodic + shutdown) ─────────────────────────────────────
 
     def save_to_repo(self):
-        """Upload entire ~/.openclaw directory → dataset .openclaw/"""
+        """Upload entire ~/.openclaw directory → dataset"""
         if not self.enabled:
             return
         if not OPENCLAW_HOME.exists():
             print("[SYNC] ~/.openclaw does not exist, nothing to save.")
             return
 
-        print(f"[SYNC] ▶ Uploading ~/.openclaw → dataset {HF_REPO_ID} ...")
+        print(f"[SYNC] ▶ Uploading ~/.openclaw → dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")
+
+        # Create a sanitized staging copy to avoid HF secret detection
+        staging_dir = None
         try:
+            staging_dir = tempfile.mkdtemp(prefix="openclaw-staging-")
+            staging_path = Path(staging_dir) / "stage"
+
+            # Copy the directory, skip symlinks and excluded patterns
+            self._copy_for_upload(OPENCLAW_HOME, staging_path)
+
+            # Sanitize openclaw.json in the staging copy
+            config_staged = staging_path / "openclaw.json"
+            if config_staged.exists():
+                self._sanitize_config_file(config_staged)
+
+            # Upload the sanitized staging directory
             self.api.upload_folder(
-                folder_path=str(OPENCLAW_HOME),
-                path_in_repo=".openclaw",
+                folder_path=str(staging_path),
+                path_in_repo=DATASET_PATH,
                 repo_id=HF_REPO_ID,
                 repo_type="dataset",
                 token=HF_TOKEN,
-                commit_message=f"Sync .openclaw — {datetime.now().isoformat()}",
+                commit_message=f"Sync openclaw_data — {datetime.now().isoformat()}",
                 ignore_patterns=EXCLUDE_PATTERNS,
             )
             print(f"[SYNC] ✓ Upload completed at {datetime.now().isoformat()}")
 
-            # Verify: list files after upload
+            # Verify
             try:
                 files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")
-                oc_files = [f for f in files if f.startswith(".openclaw/")]
-                print(f"[SYNC] Dataset now has {len(oc_files)} files under .openclaw/")
-                for f in oc_files[:20]:
+                oc_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]
+                print(f"[SYNC] Dataset now has {len(oc_files)} files under {DATASET_PATH}/")
+                for f in oc_files[:30]:
                     print(f"[SYNC]   {f}")
-                if len(oc_files) > 20:
-                    print(f"[SYNC]   ... and {len(oc_files) - 20} more")
+                if len(oc_files) > 30:
+                    print(f"[SYNC]   ... and {len(oc_files) - 30} more")
             except Exception:
                 pass
 
         except Exception as e:
             print(f"[SYNC] ✗ Upload failed: {e}")
+            traceback.print_exc()
+        finally:
+            if staging_dir and os.path.exists(staging_dir):
+                shutil.rmtree(staging_dir, ignore_errors=True)
+
+    def _copy_for_upload(self, src: Path, dst: Path):
+        """Copy directory for upload, skipping symlinks and excluded items."""
+        skip_names = {".git", "node_modules", "__pycache__", ".cache",
+                      "extensions", ".DS_Store"}
+        skip_exts = {".lock", ".tmp", ".socket", ".pid", ".pyc"}
+
+        dst.mkdir(parents=True, exist_ok=True)
+        for item in src.iterdir():
+            if item.name in skip_names:
+                continue
+            if item.is_symlink():
+                continue
+            if item.is_file():
+                if item.suffix in skip_exts:
+                    continue
+                if item.name in ("sync.log", ".persistence.lock", ".persistence-state.json"):
+                    continue
+                shutil.copy2(str(item), str(dst / item.name))
+            elif item.is_dir():
+                self._copy_for_upload(item, dst / item.name)
+
+    def _sanitize_config_file(self, config_path: Path):
+        """Sanitize openclaw.json to remove secrets before upload."""
+        try:
+            with open(config_path, "r") as f:
+                data = json.load(f)
+
+            # Replace any apiKey that looks like a real key
+            if "models" in data and "providers" in data["models"]:
+                for prov_name, prov in data["models"]["providers"].items():
+                    if isinstance(prov, dict) and "apiKey" in prov:
+                        key = prov["apiKey"]
+                        if isinstance(key, str) and not key.startswith("${"):
+                            prov["apiKey"] = "<REDACTED>"
+
+            with open(config_path, "w") as f:
+                json.dump(data, f, indent=2)
+            print("[SYNC] Sanitized openclaw.json for upload (redacted API keys)")
+        except Exception as e:
+            print(f"[SYNC] Warning: could not sanitize config: {e}")
 
     # ── Config helpers ─────────────────────────────────────────────────
 
@@ -199,7 +264,6 @@ class OpenClawFullSync:
             return
         default_src = Path(__file__).parent / "openclaw.json.default"
         if default_src.exists():
-            import shutil
             shutil.copy2(str(default_src), str(config_path))
             print("[SYNC] Created openclaw.json from default template")
         else:
@@ -239,7 +303,6 @@ class OpenClawFullSync:
                 locs = data["plugins"].get("locations", [])
                 if isinstance(locs, list) and "/dev/null" in locs:
                     data["plugins"]["locations"] = [l for l in locs if l != "/dev/null"]
-                    print("[SYNC] Removed /dev/null from plugins.locations")
 
             # Ensure agents defaults
             data.setdefault("agents", {}).setdefault("defaults", {}).setdefault("model", {})
@@ -310,7 +373,6 @@ class OpenClawFullSync:
         try:
             count = 0
             for root, dirs, files in os.walk(OPENCLAW_HOME):
-                # skip noisy dirs
                 dirs[:] = [d for d in dirs if d not in {".cache", "node_modules", "__pycache__"}]
                 for name in sorted(files):
                     rel = os.path.relpath(os.path.join(root, name), OPENCLAW_HOME)