tao-shen commited on
Commit
2f79e09
Β·
1 Parent(s): 911a0e3

fix: reduce verbose logging to prevent HF Spaces log viewer crashes

Browse files

- Suppress huggingface_hub progress bars and verbose download logs
- Remove per-file upload listing (just show summary count)
- Remove per-file dataset verification listing
- Reduce file tree debug to count only
- Filter OpenClaw process output: skip download/progress noise, forward key lines only
- All output still goes to log file for debugging

Files changed (1) hide show
  1. scripts/sync_hf.py +27 -23
scripts/sync_hf.py CHANGED
@@ -29,6 +29,14 @@ from datetime import datetime
29
  # Set timeout BEFORE importing huggingface_hub
30
  os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "300")
31
  os.environ.setdefault("HF_HUB_UPLOAD_TIMEOUT", "600")
 
 
 
 
 
 
 
 
32
 
33
  from huggingface_hub import HfApi, snapshot_download
34
 
@@ -292,17 +300,14 @@ class OpenClawFullSync:
292
  print(f"[SYNC] β–Ά Uploading ~/.openclaw β†’ dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")
293
 
294
  try:
295
- # Log what will be uploaded
296
  total_size = 0
297
  file_count = 0
298
  for root, dirs, fls in os.walk(OPENCLAW_HOME):
299
  for fn in fls:
300
  fp = os.path.join(root, fn)
301
- sz = os.path.getsize(fp)
302
- total_size += sz
303
  file_count += 1
304
- rel = os.path.relpath(fp, OPENCLAW_HOME)
305
- print(f"[SYNC] uploading: {rel} ({sz} bytes)")
306
  print(f"[SYNC] Uploading: {file_count} files, {total_size} bytes total")
307
 
308
  if file_count == 0:
@@ -327,15 +332,11 @@ class OpenClawFullSync:
327
  )
328
  print(f"[SYNC] βœ“ Upload completed at {datetime.now().isoformat()}")
329
 
330
- # Verify
331
  try:
332
  files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")
333
  oc_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]
334
  print(f"[SYNC] Dataset now has {len(oc_files)} files under {DATASET_PATH}/")
335
- for f in oc_files[:30]:
336
- print(f"[SYNC] {f}")
337
- if len(oc_files) > 30:
338
- print(f"[SYNC] ... and {len(oc_files) - 30} more")
339
  except Exception:
340
  pass
341
 
@@ -564,18 +565,9 @@ class OpenClawFullSync:
564
  traceback.print_exc()
565
 
566
  def _debug_list_files(self):
567
- print(f"[SYNC] Local ~/.openclaw tree:")
568
  try:
569
- count = 0
570
- for root, dirs, files in os.walk(OPENCLAW_HOME):
571
- dirs[:] = [d for d in dirs if d not in {".cache", "node_modules", "__pycache__"}]
572
- for name in sorted(files):
573
- rel = os.path.relpath(os.path.join(root, name), OPENCLAW_HOME)
574
- print(f"[SYNC] {rel}")
575
- count += 1
576
- if count > 50:
577
- print("[SYNC] ... (truncated)")
578
- return
579
  except Exception as e:
580
  print(f"[SYNC] listing failed: {e}")
581
 
@@ -667,13 +659,25 @@ class OpenClawFullSync:
667
  env=env # Pass environment with OPENROUTER_API_KEY
668
  )
669
 
670
- # Create a thread to copy output to both log file and stdout
671
  def copy_output():
672
  try:
673
  for line in process.stdout:
674
  log_fh.write(line)
675
  log_fh.flush()
676
- print(line, end='') # Also print to console
 
 
 
 
 
 
 
 
 
 
 
 
677
  except Exception as e:
678
  print(f"[SYNC] Output copy error: {e}")
679
  finally:
 
29
  # Set timeout BEFORE importing huggingface_hub
30
  os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "300")
31
  os.environ.setdefault("HF_HUB_UPLOAD_TIMEOUT", "600")
32
+ # Suppress huggingface_hub progress bars and verbose download/upload logs
33
+ os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
34
+ os.environ.setdefault("HF_HUB_VERBOSITY", "warning")
35
+
36
+ import logging as _logging
37
+ _logging.getLogger("huggingface_hub").setLevel(_logging.WARNING)
38
+ _logging.getLogger("huggingface_hub.utils").setLevel(_logging.WARNING)
39
+ _logging.getLogger("filelock").setLevel(_logging.WARNING)
40
 
41
  from huggingface_hub import HfApi, snapshot_download
42
 
 
300
  print(f"[SYNC] β–Ά Uploading ~/.openclaw β†’ dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")
301
 
302
  try:
303
+ # Count files to upload (no per-file logging to reduce noise)
304
  total_size = 0
305
  file_count = 0
306
  for root, dirs, fls in os.walk(OPENCLAW_HOME):
307
  for fn in fls:
308
  fp = os.path.join(root, fn)
309
+ total_size += os.path.getsize(fp)
 
310
  file_count += 1
 
 
311
  print(f"[SYNC] Uploading: {file_count} files, {total_size} bytes total")
312
 
313
  if file_count == 0:
 
332
  )
333
  print(f"[SYNC] βœ“ Upload completed at {datetime.now().isoformat()}")
334
 
335
+ # Verify (summary only)
336
  try:
337
  files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")
338
  oc_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]
339
  print(f"[SYNC] Dataset now has {len(oc_files)} files under {DATASET_PATH}/")
 
 
 
 
340
  except Exception:
341
  pass
342
 
 
565
  traceback.print_exc()
566
 
567
  def _debug_list_files(self):
 
568
  try:
569
+ count = sum(1 for _, _, files in os.walk(OPENCLAW_HOME) for _ in files)
570
+ print(f"[SYNC] Local ~/.openclaw: {count} files")
 
 
 
 
 
 
 
 
571
  except Exception as e:
572
  print(f"[SYNC] listing failed: {e}")
573
 
 
659
  env=env # Pass environment with OPENROUTER_API_KEY
660
  )
661
 
662
+ # Create a thread to copy output to log file; only print key lines to console
663
  def copy_output():
664
  try:
665
  for line in process.stdout:
666
  log_fh.write(line)
667
  log_fh.flush()
668
+ # Only forward important lines to console (errors, warnings, startup)
669
+ # Skip noisy download/progress lines that flood the HF Spaces log viewer
670
+ stripped = line.strip()
671
+ if not stripped:
672
+ continue
673
+ # Skip progress bars and download noise
674
+ if any(skip in stripped for skip in [
675
+ 'Downloading', 'Fetching', '%|', '━', '───',
676
+ 'Already cached', 'Using cache', 'tokenizer',
677
+ '.safetensors', 'model-', 'shard',
678
+ ]):
679
+ continue
680
+ print(line, end='')
681
  except Exception as e:
682
  print(f"[SYNC] Output copy error: {e}")
683
  finally: