Spaces:
Runtime error
Runtime error
fix: reduce verbose logging to prevent HF Spaces log viewer crashes
Browse files- Suppress huggingface_hub progress bars and verbose download logs
- Remove per-file upload listing (just show summary count)
- Remove per-file dataset verification listing
- Reduce file tree debug to count only
- Filter OpenClaw process output: skip download/progress noise, forward key lines only
- All output still goes to log file for debugging
- scripts/sync_hf.py +27 -23
scripts/sync_hf.py
CHANGED
|
@@ -29,6 +29,14 @@ from datetime import datetime
|
|
| 29 |
# Set timeout BEFORE importing huggingface_hub
|
| 30 |
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "300")
|
| 31 |
os.environ.setdefault("HF_HUB_UPLOAD_TIMEOUT", "600")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
from huggingface_hub import HfApi, snapshot_download
|
| 34 |
|
|
@@ -292,17 +300,14 @@ class OpenClawFullSync:
|
|
| 292 |
print(f"[SYNC] βΆ Uploading ~/.openclaw β dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")
|
| 293 |
|
| 294 |
try:
|
| 295 |
-
#
|
| 296 |
total_size = 0
|
| 297 |
file_count = 0
|
| 298 |
for root, dirs, fls in os.walk(OPENCLAW_HOME):
|
| 299 |
for fn in fls:
|
| 300 |
fp = os.path.join(root, fn)
|
| 301 |
-
|
| 302 |
-
total_size += sz
|
| 303 |
file_count += 1
|
| 304 |
-
rel = os.path.relpath(fp, OPENCLAW_HOME)
|
| 305 |
-
print(f"[SYNC] uploading: {rel} ({sz} bytes)")
|
| 306 |
print(f"[SYNC] Uploading: {file_count} files, {total_size} bytes total")
|
| 307 |
|
| 308 |
if file_count == 0:
|
|
@@ -327,15 +332,11 @@ class OpenClawFullSync:
|
|
| 327 |
)
|
| 328 |
print(f"[SYNC] β Upload completed at {datetime.now().isoformat()}")
|
| 329 |
|
| 330 |
-
# Verify
|
| 331 |
try:
|
| 332 |
files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")
|
| 333 |
oc_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]
|
| 334 |
print(f"[SYNC] Dataset now has {len(oc_files)} files under {DATASET_PATH}/")
|
| 335 |
-
for f in oc_files[:30]:
|
| 336 |
-
print(f"[SYNC] {f}")
|
| 337 |
-
if len(oc_files) > 30:
|
| 338 |
-
print(f"[SYNC] ... and {len(oc_files) - 30} more")
|
| 339 |
except Exception:
|
| 340 |
pass
|
| 341 |
|
|
@@ -564,18 +565,9 @@ class OpenClawFullSync:
|
|
| 564 |
traceback.print_exc()
|
| 565 |
|
| 566 |
def _debug_list_files(self):
|
| 567 |
-
print(f"[SYNC] Local ~/.openclaw tree:")
|
| 568 |
try:
|
| 569 |
-
count =
|
| 570 |
-
|
| 571 |
-
dirs[:] = [d for d in dirs if d not in {".cache", "node_modules", "__pycache__"}]
|
| 572 |
-
for name in sorted(files):
|
| 573 |
-
rel = os.path.relpath(os.path.join(root, name), OPENCLAW_HOME)
|
| 574 |
-
print(f"[SYNC] {rel}")
|
| 575 |
-
count += 1
|
| 576 |
-
if count > 50:
|
| 577 |
-
print("[SYNC] ... (truncated)")
|
| 578 |
-
return
|
| 579 |
except Exception as e:
|
| 580 |
print(f"[SYNC] listing failed: {e}")
|
| 581 |
|
|
@@ -667,13 +659,25 @@ class OpenClawFullSync:
|
|
| 667 |
env=env # Pass environment with OPENROUTER_API_KEY
|
| 668 |
)
|
| 669 |
|
| 670 |
-
# Create a thread to copy output to
|
| 671 |
def copy_output():
|
| 672 |
try:
|
| 673 |
for line in process.stdout:
|
| 674 |
log_fh.write(line)
|
| 675 |
log_fh.flush()
|
| 676 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 677 |
except Exception as e:
|
| 678 |
print(f"[SYNC] Output copy error: {e}")
|
| 679 |
finally:
|
|
|
|
| 29 |
# Set timeout BEFORE importing huggingface_hub
|
| 30 |
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "300")
|
| 31 |
os.environ.setdefault("HF_HUB_UPLOAD_TIMEOUT", "600")
|
| 32 |
+
# Suppress huggingface_hub progress bars and verbose download/upload logs
|
| 33 |
+
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
| 34 |
+
os.environ.setdefault("HF_HUB_VERBOSITY", "warning")
|
| 35 |
+
|
| 36 |
+
import logging as _logging
|
| 37 |
+
_logging.getLogger("huggingface_hub").setLevel(_logging.WARNING)
|
| 38 |
+
_logging.getLogger("huggingface_hub.utils").setLevel(_logging.WARNING)
|
| 39 |
+
_logging.getLogger("filelock").setLevel(_logging.WARNING)
|
| 40 |
|
| 41 |
from huggingface_hub import HfApi, snapshot_download
|
| 42 |
|
|
|
|
| 300 |
print(f"[SYNC] βΆ Uploading ~/.openclaw β dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")
|
| 301 |
|
| 302 |
try:
|
| 303 |
+
# Count files to upload (no per-file logging to reduce noise)
|
| 304 |
total_size = 0
|
| 305 |
file_count = 0
|
| 306 |
for root, dirs, fls in os.walk(OPENCLAW_HOME):
|
| 307 |
for fn in fls:
|
| 308 |
fp = os.path.join(root, fn)
|
| 309 |
+
total_size += os.path.getsize(fp)
|
|
|
|
| 310 |
file_count += 1
|
|
|
|
|
|
|
| 311 |
print(f"[SYNC] Uploading: {file_count} files, {total_size} bytes total")
|
| 312 |
|
| 313 |
if file_count == 0:
|
|
|
|
| 332 |
)
|
| 333 |
print(f"[SYNC] β Upload completed at {datetime.now().isoformat()}")
|
| 334 |
|
| 335 |
+
# Verify (summary only)
|
| 336 |
try:
|
| 337 |
files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")
|
| 338 |
oc_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]
|
| 339 |
print(f"[SYNC] Dataset now has {len(oc_files)} files under {DATASET_PATH}/")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
except Exception:
|
| 341 |
pass
|
| 342 |
|
|
|
|
| 565 |
traceback.print_exc()
|
| 566 |
|
| 567 |
def _debug_list_files(self):
|
|
|
|
| 568 |
try:
|
| 569 |
+
count = sum(1 for _, _, files in os.walk(OPENCLAW_HOME) for _ in files)
|
| 570 |
+
print(f"[SYNC] Local ~/.openclaw: {count} files")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 571 |
except Exception as e:
|
| 572 |
print(f"[SYNC] listing failed: {e}")
|
| 573 |
|
|
|
|
| 659 |
env=env # Pass environment with OPENROUTER_API_KEY
|
| 660 |
)
|
| 661 |
|
| 662 |
+
# Create a thread to copy output to log file; only print key lines to console
|
| 663 |
def copy_output():
|
| 664 |
try:
|
| 665 |
for line in process.stdout:
|
| 666 |
log_fh.write(line)
|
| 667 |
log_fh.flush()
|
| 668 |
+
# Only forward important lines to console (errors, warnings, startup)
|
| 669 |
+
# Skip noisy download/progress lines that flood the HF Spaces log viewer
|
| 670 |
+
stripped = line.strip()
|
| 671 |
+
if not stripped:
|
| 672 |
+
continue
|
| 673 |
+
# Skip progress bars and download noise
|
| 674 |
+
if any(skip in stripped for skip in [
|
| 675 |
+
'Downloading', 'Fetching', '%|', 'β', 'βββ',
|
| 676 |
+
'Already cached', 'Using cache', 'tokenizer',
|
| 677 |
+
'.safetensors', 'model-', 'shard',
|
| 678 |
+
]):
|
| 679 |
+
continue
|
| 680 |
+
print(line, end='')
|
| 681 |
except Exception as e:
|
| 682 |
print(f"[SYNC] Output copy error: {e}")
|
| 683 |
finally:
|