tao-shen commited on
Commit
96469d3
·
1 Parent(s): 875f481

feat: sync-on-change optimization & update config to GLM-4.7-FlashX

Browse files
Files changed (1) hide show
  1. scripts/sync_hf.py +58 -47
scripts/sync_hf.py CHANGED
@@ -40,44 +40,10 @@ class OpenClawHFSync:
40
 
41
  self.enabled = True
42
  self.api = HfApi(token=HF_TOKEN)
 
43
  self._ensure_repo()
44
 
45
- def _upload_folder(self, local_dir, remote_path, msg, ignore=None):
46
- if not local_dir.exists(): return
47
- patterns = ignore or EXCLUDE_PATTERNS
48
-
49
- # Debug: list files being considered for credentials to ensure they are found
50
- if "credentials" in str(local_dir):
51
- try:
52
- files = list(local_dir.glob("**/*"))
53
- print(f"[SYNC] Scanning credentials dir: {local_dir}, found {len(files)} items")
54
- for f in files:
55
- if f.is_file():
56
- print(f"[SYNC] - {f.relative_to(local_dir)}")
57
- except Exception:
58
- pass
59
 
60
- try:
61
- self.api.upload_folder(
62
- folder_path=str(local_dir),
63
- path_in_repo=remote_path,
64
- repo_id=HF_REPO_ID,
65
- repo_type="dataset",
66
- token=HF_TOKEN,
67
- commit_message=msg,
68
- commit_description=f"Automated sync at {datetime.now().isoformat()}",
69
- ignore_patterns=patterns,
70
- )
71
- except Exception as e:
72
- print(f"[SYNC] Upload folder error ({local_dir}): {e}")
73
- if not HF_TOKEN:
74
- print("[SYNC] WARNING: HF_TOKEN not set in environment. Persistence disabled.")
75
- self.enabled = False
76
- return
77
-
78
- self.enabled = True
79
- self.api = HfApi(token=HF_TOKEN)
80
- self._ensure_repo()
81
 
82
  def _ensure_repo(self):
83
  try:
@@ -112,6 +78,13 @@ class OpenClawHFSync:
112
 
113
  print("[SYNC] Load complete.")
114
  self._debug_list_files()
 
 
 
 
 
 
 
115
 
116
  def _patch_config(self):
117
  """Ensure critical config settings are present and valid after restore."""
@@ -151,22 +124,30 @@ class OpenClawHFSync:
151
  print("[SYNC] Removed '/dev/null' from plugins.locations")
152
  modified = True
153
 
154
- # 2. Enforce Model to GLM-4-Plus (User's desired state from context)
155
- # Check if we need to add the provider as well
156
  if 'agents' not in data: data['agents'] = {}
157
  if 'defaults' not in data['agents']: data['agents']['defaults'] = {}
158
  if 'model' not in data['agents']['defaults']: data['agents']['defaults']['model'] = {}
159
 
160
- current_model = data['agents']['defaults']['model'].get('primary')
161
- if current_model != 'zhipu/glm-4-plus':
162
- print(f"[SYNC] Updating model.primary: {current_model} -> zhipu/glm-4-plus")
163
- data['agents']['defaults']['model']['primary'] = 'zhipu/glm-4-plus'
164
- modified = True
165
-
166
- # Ensure Zhipu provider exists if missing (basic check)
167
- # This is complex to patch fully, but we ensure 'zhipu' key exists in providers if needed
168
- # For now, relying on User to have set it up or defaults.
 
 
 
 
169
 
 
 
 
 
 
170
  if modified:
171
  with open(config_path, 'w') as f:
172
  json.dump(data, f, indent=2)
@@ -177,6 +158,26 @@ class OpenClawHFSync:
177
  except Exception as e:
178
  print(f"[SYNC] Failed to patch config: {e}")
179
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
180
  def _download_config(self):
181
  config_local = OPENCLAW_HOME / "openclaw.json"
182
  try:
@@ -331,7 +332,17 @@ class OpenClawHFSync:
331
  print(f"[SYNC] Upload file error ({local_path}): {e}")
332
 
333
  def _upload_folder(self, local_dir, remote_path, msg, ignore=None):
334
- if not local_dir.exists(): return
 
 
 
 
 
 
 
 
 
 
335
  patterns = ignore or EXCLUDE_PATTERNS
336
 
337
  # Debug: list files being considered for credentials to ensure they are found
 
40
 
41
  self.enabled = True
42
  self.api = HfApi(token=HF_TOKEN)
43
+ self.checksums = {}
44
  self._ensure_repo()
45
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
 
48
  def _ensure_repo(self):
49
  try:
 
78
 
79
  print("[SYNC] Load complete.")
80
  self._debug_list_files()
81
+ self._init_checksums()
82
+
83
+ def _init_checksums(self):
84
+ for name in ["credentials", "workspace", "agents"]:
85
+ p = OPENCLAW_HOME / name
86
+ if p.exists():
87
+ self.checksums[str(p)] = self._get_dir_checksum(p)
88
 
89
  def _patch_config(self):
90
  """Ensure critical config settings are present and valid after restore."""
 
124
  print("[SYNC] Removed '/dev/null' from plugins.locations")
125
  modified = True
126
 
127
+ # 2. Enforce Model to GLM-4-Flash (User's desired state from context)
 
128
  if 'agents' not in data: data['agents'] = {}
129
  if 'defaults' not in data['agents']: data['agents']['defaults'] = {}
130
  if 'model' not in data['agents']['defaults']: data['agents']['defaults']['model'] = {}
131
 
132
+ # Update Provider Zhipu
133
+ if 'models' not in data: data['models'] = {}
134
+ if 'providers' not in data['models']: data['models']['providers'] = {}
135
+
136
+ # Force Zhipu config
137
+ data['models']['providers']['zhipu'] = {
138
+ "baseUrl": "https://api.z.ai/api/paas/v4",
139
+ "apiKey": "${ZHIPU_API_KEY}",
140
+ "api": "openai-chat",
141
+ "models": [
142
+ { "id": "GLM-4.7-FlashX", "name": "GLM-4.7 FlashX" }
143
+ ]
144
+ }
145
 
146
+ # Set default model
147
+ print(f"[SYNC] Updating model.primary -> zhipu/GLM-4.7-FlashX")
148
+ data['agents']['defaults']['model']['primary'] = 'zhipu/GLM-4.7-FlashX'
149
+ modified = True
150
+
151
  if modified:
152
  with open(config_path, 'w') as f:
153
  json.dump(data, f, indent=2)
 
158
  except Exception as e:
159
  print(f"[SYNC] Failed to patch config: {e}")
160
 
161
+ def _get_dir_checksum(self, directory, patterns=None):
162
+ """Calculate a simple checksum of a directory based on mtime and size of files."""
163
+ if not directory.exists(): return None
164
+ import hashlib
165
+ hash_md5 = hashlib.md5()
166
+ # TODO: Implement pattern filtering for checksum if needed,
167
+ # but for now simple walk is enough as upload_folder handles ignores.
168
+ # We just want to avoid calling upload_folder if nothing changed.
169
+ for root, dirs, files in os.walk(directory):
170
+ for name in sorted(files):
171
+ if name.endswith(".pyc") or name == "models.json": continue # meaningful ignores
172
+ fp = Path(root) / name
173
+ try:
174
+ stats = fp.stat()
175
+ # Hash path, mtime, size
176
+ s = f"{fp.relative_to(directory)}:{stats.st_mtime}:{stats.st_size}".encode('utf-8')
177
+ hash_md5.update(s)
178
+ except: pass
179
+ return hash_md5.hexdigest()
180
+
181
  def _download_config(self):
182
  config_local = OPENCLAW_HOME / "openclaw.json"
183
  try:
 
332
  print(f"[SYNC] Upload file error ({local_path}): {e}")
333
 
334
  def _upload_folder(self, local_dir, remote_path, msg, ignore=None):
335
+ # Checksum check
336
+ current_sum = self._get_dir_checksum(local_dir)
337
+ last_sum = self.checksums.get(str(local_dir))
338
+
339
+ if current_sum == last_sum:
340
+ # print(f"[SYNC] Skipping {local_dir.name} (no changes)")
341
+ return
342
+
343
+ print(f"[SYNC] Changes detected in {local_dir.name}, syncing...")
344
+ self.checksums[str(local_dir)] = current_sum
345
+
346
  patterns = ignore or EXCLUDE_PATTERNS
347
 
348
  # Debug: list files being considered for credentials to ensure they are found