sharween commited on
Commit
ec8d6ef
·
verified ·
1 Parent(s): bed1d28

Upload backup-manager.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. backup-manager.py +200 -200
backup-manager.py CHANGED
@@ -1,25 +1,25 @@
1
  #!/usr/bin/env python3
2
  """
3
- backup-manager.py -- WebDAV + HF Dataset ?????/??
4
-
5
- ????:
6
- +------------------+----------+--------------------------+--------------------------+
7
- | ?? | ?? | ?? | ?? |
8
- +------------------+----------+--------------------------+--------------------------+
9
- | WebDAV | ??+??| BACKUP_WEBDAV_INTERVAL | WEBDAV_MAX_BACKUPS ? |
10
- | | | (?? 60 ?) | (?? 30 ?) |
11
- +------------------+----------+--------------------------+--------------------------+
12
- | HF Dataset | ??? | BACKUP_HF_INTERVAL_SMALL | 1 ? (??) |
13
- | | | (?? 30 ?) | |
14
- +------------------+----------+--------------------------+--------------------------+
15
- | HF Dataset | ??? | BACKUP_HF_INTERVAL_BIG | DATASET_MAX_BACKUPS ? |
16
- | | | (?? 30 ?) | (?? 30 ?) |
17
- +------------------+----------+--------------------------+--------------------------+
18
-
19
- ????: HF Dataset ?? 1GB, ?????
20
- ??: WebDAV -> HF Dataset ??
21
-
22
- ????: ???????? last_run ??, ???????
23
  """
24
 
25
  import hashlib, json, os, tarfile, time, sys
@@ -28,13 +28,13 @@ from pathlib import Path
28
 
29
  import requests
30
 
31
- # ???????????????????????????????????????????????????????????????????????????????
32
- # ??
33
- # ???????????????????????????????????????????????????????????????????????????????
34
 
35
  STATE_DIR = os.environ.get("OPENCLAW_STATE_DIR", "/root/.openclaw")
36
 
37
- # -- WebDAV --
38
  WEBDAV_URL = os.environ.get("WEBDAV_URL", "").rstrip("/")
39
  WEBDAV_USER = os.environ.get("WEBDAV_USERNAME", "")
40
  WEBDAV_PASS = os.environ.get("WEBDAV_PASSWORD", "")
@@ -42,64 +42,64 @@ WEBDAV_BACKUP_DIR = os.environ.get("WEBDAV_BACKUP_DIR", "openclaw-backup")
42
  WEBDAV_INTERVAL = int(os.environ.get("BACKUP_WEBDAV_INTERVAL", "60"))
43
  WEBDAV_MAX_BACKUPS = int(os.environ.get("WEBDAV_MAX_BACKUPS", "30"))
44
 
45
- # -- HF Dataset --
46
  HF_REPO = os.environ.get("HF_DATASET", "")
47
  HF_TOKEN = os.environ.get("HF_TOKEN", "")
48
  HF_SMALL_INTERVAL = int(os.environ.get("BACKUP_HF_INTERVAL_SMALL", "30"))
49
  HF_BIG_INTERVAL = int(os.environ.get("BACKUP_HF_INTERVAL_BIG", "30"))
50
  DATASET_MAX_BACKUPS = int(os.environ.get("DATASET_MAX_BACKUPS", "30"))
51
 
52
- # -- ????? --
53
  FULL_NAME = "openclaw-full.tar.gz"
54
  MANIFEST_NAME = "_incremental_manifest.json"
55
 
56
- # -- HF ???? --
57
  MAX_HF_STORAGE = 1 * 1024 * 1024 * 1024 # 1 GB
58
 
59
- # -- ???? --
60
- HASH_CHUNK_SIZE = 65536 # SHA256 ??????
61
- WD_CHECK_TIMEOUT = 3 # WebDAV ???????
62
- WD_REQ_TIMEOUT = 30 # WebDAV ??????
63
 
64
- # -- ???? --
65
  RED = "\033[91m"
66
  GREEN = "\033[92m"
67
  YELLOW = "\033[93m"
68
  RESET = "\033[0m"
69
 
70
- # ???????????????????????????????????????????????????????????????????????????????
71
- # ???????????????
72
- # ???????????????????????????????????????????????????????????????????????????????
73
 
74
  def _print_banner():
75
- """????????????"""
76
  global WD_SKIPPED, HF_SKIPPED
77
- sep = "-" * 50
78
- print(f"[backup] +{sep}+")
79
 
80
  wd_configured = _wd_is_configured()
81
  if not wd_configured:
82
- print(f"[backup] | WebDAV {RED}[FAIL] ??: ?? WEBDAV_URL / WEBDAV_USERNAME / WEBDAV_PASSWORD / WEBDAV_BACKUP_DIR ???????{RESET}")
83
  WD_SKIPPED = True
84
  else:
85
- print(f"[backup] | WebDAV {GREEN}[OK] ???{RESET} ??={WEBDAV_INTERVAL}? ??={WEBDAV_MAX_BACKUPS}?")
86
  WD_SKIPPED = False
87
 
88
  if not HF_REPO or not HF_TOKEN:
89
  hf_missing = []
90
  if not HF_REPO: hf_missing.append("HF_DATASET")
91
  if not HF_TOKEN: hf_missing.append("HF_TOKEN")
92
- print(f"[backup] | HF??? {RED}[FAIL] ??: ?? {', '.join(hf_missing)}{RESET}")
93
- print(f"[backup] | HF??? {RED}[FAIL] ??{RESET}")
94
  HF_SKIPPED = True
95
  else:
96
- print(f"[backup] | HF??? {GREEN}[OK] ???{RESET} ??={HF_SMALL_INTERVAL}?")
97
- print(f"[backup] | HF??? {GREEN}[OK] ???{RESET} ??={HF_BIG_INTERVAL}? ??={DATASET_MAX_BACKUPS}?")
98
  HF_SKIPPED = False
99
 
100
- print(f"[backup] +{sep}+")
101
 
102
- # ???? banner ????? import ??????????
103
  _DISPLAYED_BANNER = False
104
 
105
  def _ensure_banner():
@@ -108,12 +108,12 @@ def _ensure_banner():
108
  _print_banner()
109
  _DISPLAYED_BANNER = True
110
 
111
- # ???????????????????????????????????????????????????????????????????????????????
112
- # ????
113
- # ???????????????????????????????????????????????????????????????????????????????
114
 
115
  def _create_tar() -> str:
116
- """?? STATE_DIR ? tar.gz, ??????."""
117
  tarpath = f"/tmp/openclaw-backup-{int(time.time()*1000)}.tar.gz"
118
  try:
119
  with tarfile.open(tarpath, "w:gz") as tar:
@@ -127,22 +127,22 @@ def _create_tar() -> str:
127
  try:
128
  tar.add(str(item), arcname=name)
129
  except (PermissionError, FileNotFoundError) as e:
130
- print(f"[backup] ? ???? {name}: {e}")
131
  continue
132
  else:
133
- print(f"[backup] ? STATE_DIR {STATE_DIR} ???, ?????")
134
  root.mkdir(parents=True, exist_ok=True)
135
  size = os.path.getsize(tarpath)
136
  print(f"[backup] Archive created ({size/1024/1024:.1f} MB)")
137
  except (OSError, tarfile.TarError) as e:
138
- print(f"[backup] ? ??????: {e}")
139
  if os.path.exists(tarpath):
140
  os.remove(tarpath)
141
- raise # ??????
142
  return tarpath
143
 
144
  def _file_hash(path: str) -> str:
145
- """????? SHA256 ??, ??????????."""
146
  try:
147
  h = hashlib.sha256()
148
  with open(path, "rb") as f:
@@ -153,15 +153,15 @@ def _file_hash(path: str) -> str:
153
  h.update(chunk)
154
  return h.hexdigest()
155
  except (OSError, PermissionError, FileNotFoundError) as e:
156
- print(f"[backup] ? ????????: {path} -> {e}")
157
  return ""
158
 
159
  def _today_str() -> str:
160
  return time.strftime("%Y%m%d")
161
 
162
- # ???????????????????????????????????????????????????????????????????????????????
163
- # WebDAV ?? HTTP ?
164
- # ???????????????????????????????????????????????????????????????????????????????
165
 
166
  def _wd_auth():
167
  return (WEBDAV_USER, WEBDAV_PASS) if WEBDAV_USER else None
@@ -171,17 +171,17 @@ def _wd_url(path=""):
171
 
172
  def _wd_req(method, path="", **kwargs):
173
  url = _wd_url(path)
174
- kwargs.setdefault("timeout", WD_REQ_TIMEOUT) # ?????/????????
175
  resp = requests.request(method, url, auth=_wd_auth(), **kwargs)
176
  resp.raise_for_status()
177
  return resp
178
 
179
  def _wd_check():
180
- """???? WebDAV ??? (3s ??), ???? False"""
181
  try:
182
  url = _wd_url("")
183
  resp = requests.request("PROPFIND", url, auth=_wd_auth(), timeout=3)
184
- resp.raise_for_status() # 401/403/404 ??????
185
  return True
186
  except Exception:
187
  return False
@@ -200,16 +200,16 @@ def wd_download(path):
200
  return _wd_req("GET", path).content
201
 
202
  def wd_delete(path):
203
- """?? WebDAV ????"""
204
  try:
205
  _wd_req("DELETE", path)
206
  return True
207
  except Exception as e:
208
- print(f"[backup] ? WebDAV ????: {path} -> {e}")
209
  return False
210
 
211
  def wd_mkdir(parts):
212
- """???? WebDAV ??"""
213
  try:
214
  _wd_req("MKCOL", "")
215
  except Exception:
@@ -222,7 +222,7 @@ def wd_mkdir(parts):
222
  pass
223
 
224
  def wd_list(path=""):
225
- """?? WebDAV ?????????"""
226
  try:
227
  import xml.etree.ElementTree as ET
228
  resp = requests.request(
@@ -242,12 +242,12 @@ def wd_list(path=""):
242
  items.append(name)
243
  return [x for x in items if x != WEBDAV_BACKUP_DIR.split("/")[-1]]
244
  except Exception as e:
245
- print(f"[backup] ? WebDAV ??????: {e}")
246
  return []
247
 
248
- # ???????????????????????????????????????????????????????????????????????????????
249
- # WebDAV ???? (Manifest)
250
- # ???????????????????????????????????????????????????????????????????????????????
251
 
252
  def _load_manifest() -> dict:
253
  try:
@@ -260,7 +260,7 @@ def _save_manifest(manifest: dict):
260
  wd_upload(MANIFEST_NAME, json.dumps(manifest, indent=2).encode())
261
 
262
  def _scan_and_upload_changes(root: Path, manifest: dict) -> int:
263
- """?????????? WebDAV. ???????."""
264
  changed = 0
265
  for fpath in root.rglob("*"):
266
  if not fpath.is_file():
@@ -292,7 +292,7 @@ def _scan_and_upload_changes(root: Path, manifest: dict) -> int:
292
  changed += 1
293
  except Exception as e:
294
  err_str = str(e)
295
- # 409 = ????????????????
296
  if "409" in err_str or "Conflict" in err_str:
297
  manifest[rel] = {
298
  "sha256": cur_h,
@@ -301,24 +301,24 @@ def _scan_and_upload_changes(root: Path, manifest: dict) -> int:
301
  }
302
  changed += 1
303
  else:
304
- print(f"[backup] ? ?????? (??): {rel} -> {e}")
305
  return changed
306
 
307
 
308
  def incremental_backup() -> int:
309
- """??????, ??? WebDAV. ???????."""
310
  root = Path(STATE_DIR)
311
  if not root.exists():
312
  return 0
313
 
314
- # ?? WebDAV ?????
315
  if not _wd_is_configured():
316
- print(f"[backup] ? WebDAV ?????, ??????")
317
  return 0
318
 
319
- # ???? WebDAV ???
320
  if not _wd_check():
321
- print(f"[backup] ? WebDAV ???, ??????")
322
  return 0
323
 
324
  manifest = _load_manifest()
@@ -328,15 +328,15 @@ def incremental_backup() -> int:
328
  try:
329
  _save_manifest(manifest)
330
  except Exception as e:
331
- print(f"[backup] ? Manifest ????: {e}")
332
  return changed
333
 
334
- # ???????????????????????????????????????????????????????????????????????????????
335
- # WebDAV ???? + ??
336
- # ???????????????????????????????????????????????????????????????????????????????
337
 
338
  def _wd_cleanup():
339
- """???? WEBDAV_MAX_BACKUPS ?? WebDAV ??????"""
340
  files = wd_list("")
341
  now = datetime.now()
342
  deleted = 0
@@ -344,32 +344,32 @@ def _wd_cleanup():
344
  if not f.startswith("openclaw-full-") or not f.endswith(".tar.gz"):
345
  continue
346
  if f == FULL_NAME:
347
- continue # ???????
348
  try:
349
  date_str = f.replace("openclaw-full-", "").replace(".tar.gz", "")
350
  dt = datetime.strptime(date_str, "%Y%m%d")
351
  days_old = (now - dt).days
352
  if days_old > WEBDAV_MAX_BACKUPS:
353
  if wd_delete(f):
354
- print(f"[backup] WebDAV ??????: {f} ({days_old}??)")
355
  deleted += 1
356
  except ValueError:
357
  continue
358
  if deleted:
359
- print(f"[backup] WebDAV ??? {deleted} ???????")
360
  return deleted
361
 
362
  def _wd_is_configured() -> bool:
363
- """?? WebDAV ?? 4 ???????"""
364
  return bool(WEBDAV_URL and WEBDAV_USER and WEBDAV_PASS and WEBDAV_BACKUP_DIR)
365
 
366
  def full_backup():
367
- """?? tar -> WebDAV (???: openclaw-full-YYYYMMDD.tar.gz)"""
368
  tarpath = _create_tar()
369
  if _wd_is_configured():
370
- # ???? WebDAV ???
371
  if not _wd_check():
372
- print(f"[backup] ? WebDAV ???, ??????")
373
  os.remove(tarpath)
374
  return
375
  wd_mkdir([])
@@ -377,26 +377,26 @@ def full_backup():
377
  try:
378
  with open(tarpath, "rb") as f:
379
  wd_upload(filename, f.read())
380
- print(f"[backup] WebDAV ???????? -> {filename}")
381
- # ???????????????????????
382
  with open(tarpath, "rb") as f:
383
  wd_upload(FULL_NAME, f.read())
384
- # ????
385
  _wd_cleanup()
386
  except Exception as e:
387
- print(f"[backup] ? WebDAV ????????: {e}")
388
  os.remove(tarpath)
389
 
390
- # ???????????????????????????????????????????????????????????????????????????????
391
- # HF Dataset ???
392
- # ???????????????????????????????????????????????????????????????????????????????
393
 
394
  def _hf_list_backup_files() -> dict | None:
395
  """
396
- ?? HF Dataset ????????.
397
- ??: {???: ??(bytes)} ??
398
- {} ?????
399
- None API ????
400
  """
401
  if not HF_REPO or not HF_TOKEN:
402
  return {}
@@ -405,7 +405,7 @@ def _hf_list_backup_files() -> dict | None:
405
  try:
406
  api = HfApi(timeout=60)
407
  except TypeError:
408
- api = HfApi() # ?????? timeout ??
409
  backups = {}
410
  for entry in api.list_repo_tree(HF_REPO, repo_type="dataset", token=HF_TOKEN):
411
  name = entry.rfilename
@@ -413,29 +413,29 @@ def _hf_list_backup_files() -> dict | None:
413
  backups[name] = entry.size if hasattr(entry, "size") else 0
414
  return backups
415
  except Exception as e:
416
- print(f"[backup] ? HF Dataset list API ??: {e}")
417
- return None # ??? "?????" (?? {})
418
 
419
  def hf_small_backup():
420
- """???: tar.gz -> openclaw-full.tar.gz (??), ?????????"""
421
  if not HF_REPO or not HF_TOKEN:
422
  return
423
 
424
  tarpath = _create_tar()
425
  tar_size = os.path.getsize(tarpath)
426
 
427
- # -- ???? --
428
  backups = _hf_list_backup_files()
429
- if backups is None: # API ????????????
430
- print(f"[backup] ? HF?????: Dataset API ???")
431
  os.remove(tarpath)
432
  return
433
  existing_total = sum(backups.values())
434
 
435
  if existing_total + tar_size > MAX_HF_STORAGE:
436
- print(f"{RED}[backup] ? HF?????: 1GB????! "
437
- f"(?? {existing_total/1024/1024:.1f}MB + "
438
- f"??? {tar_size/1024/1024:.1f}MB > 1GB){RESET}")
439
  os.remove(tarpath)
440
  return
441
 
@@ -444,7 +444,7 @@ def hf_small_backup():
444
  try:
445
  api = HfApi(timeout=60)
446
  except TypeError:
447
- api = HfApi() # ?????? timeout ??
448
  with open(tarpath, "rb") as f:
449
  api.upload_file(
450
  path_or_fileobj=f,
@@ -453,21 +453,21 @@ def hf_small_backup():
453
  repo_type="dataset",
454
  token=HF_TOKEN,
455
  )
456
- print(f"[backup] ? HF??????? -> {FULL_NAME}")
457
  except Exception as e:
458
- print(f"{RED}[backup] ? HF???????: {e}{RESET}")
459
  finally:
460
  if os.path.exists(tarpath):
461
  os.remove(tarpath)
462
 
463
- # ???????????????????????????????????????????????????????????????????????????????
464
- # HF Dataset ??? + ??
465
- # ???????????????????????????????????????????????????????????????????????????????
466
 
467
  def _hf_cleanup():
468
- """???? DATASET_MAX_BACKUPS ?? HF ?????"""
469
  backups = _hf_list_backup_files()
470
- if not backups: # None(API??) ? {} (???) ???
471
  return 0
472
 
473
  from huggingface_hub import HfApi
@@ -489,21 +489,21 @@ def _hf_cleanup():
489
  repo_type="dataset",
490
  token=HF_TOKEN,
491
  )
492
- print(f"[backup] HF ???????: {name} ({days_old}??)")
493
  deleted += 1
494
  except (ValueError, Exception):
495
  continue
496
 
497
  if deleted:
498
- print(f"[backup] HF ??? {deleted} ??????")
499
  return deleted
500
 
501
  def _hf_make_space(backups: dict, tar_size: int, api) -> bool:
502
- """????????????, ?????? True, ???? False."""
503
- print(f"[backup] ? HF??????? "
504
- f"(?? {sum(backups.values())/1024/1024:.1f}MB + "
505
- f"??? {tar_size/1024/1024:.1f}MB > 1GB)")
506
- print(f"[backup] -> ????????????...")
507
 
508
  dated_big = []
509
  for name, sz in backups.items():
@@ -525,24 +525,24 @@ def _hf_make_space(backups: dict, tar_size: int, api) -> bool:
525
  repo_type="dataset",
526
  token=HF_TOKEN,
527
  )
528
- print(f"[backup] ?? {name} (?? {ds})")
529
  del backups[name]
530
  if sum(backups.values()) + tar_size <= MAX_HF_STORAGE:
531
  return True
532
  except Exception:
533
  continue
534
 
535
- print(f"{RED}[backup] ? HF?????: ???????????1GB "
536
- f"(? {tar_size/1024/1024:.1f}MB > 1GB){RESET}")
537
  return False
538
 
539
 
540
  def _hf_upload_backup(tarpath: str, tar_size: int, filename: str, api) -> None:
541
- """?????? HF Dataset??????????."""
542
- # -- ???? & ????? --
543
  backups = _hf_list_backup_files()
544
- if backups is None: # API ???????????
545
- print(f"[backup] ? HF?????: Dataset API ???")
546
  os.remove(tarpath)
547
  return
548
  existing_total = sum(backups.values())
@@ -551,12 +551,12 @@ def _hf_upload_backup(tarpath: str, tar_size: int, filename: str, api) -> None:
551
  if not _hf_make_space(backups, tar_size, api):
552
  os.remove(tarpath)
553
  return
554
- print(f"[backup] -> ?????, ???????")
555
- backups = _hf_list_backup_files() # ??????????
556
  if backups is None:
557
  backups = {}
558
 
559
- # -- ??????????, ?????? --
560
  if filename in backups:
561
  try:
562
  api.delete_file(
@@ -566,9 +566,9 @@ def _hf_upload_backup(tarpath: str, tar_size: int, filename: str, api) -> None:
566
  token=HF_TOKEN,
567
  )
568
  except Exception:
569
- print(f"[backup] ? ?????????? (??): {filename}")
570
 
571
- # -- ?? --
572
  with open(tarpath, "rb") as f:
573
  api.upload_file(
574
  path_or_fileobj=f,
@@ -577,16 +577,16 @@ def _hf_upload_backup(tarpath: str, tar_size: int, filename: str, api) -> None:
577
  repo_type="dataset",
578
  token=HF_TOKEN,
579
  )
580
- print(f"[backup] ? HF??????? -> {filename}")
581
 
582
- # -- ???? --
583
  _hf_cleanup()
584
 
585
 
586
  def hf_large_backup():
587
  """
588
- ???: tar.gz -> openclaw-full-YYYYMMDD.tar.gz
589
- ?????????????????, ????????
590
  """
591
  if not HF_REPO or not HF_TOKEN:
592
  return
@@ -608,20 +608,20 @@ def hf_large_backup():
608
  try:
609
  api = HfApi(timeout=60)
610
  except TypeError:
611
- api = HfApi() # ?????? timeout ??
612
  _hf_upload_backup(tarpath, tar_size, filename, api)
613
  except Exception as e:
614
- print(f"{RED}[backup] ? HF???????: {e}{RESET}")
615
  finally:
616
  if os.path.exists(tarpath):
617
  os.remove(tarpath)
618
 
619
- # ???????????????????????????????????????????????????????????????????????????????
620
- # ??
621
- # ???????????????????????????????????????????????????????????????????????????????
622
 
623
  def _hf_download(filename: str = FULL_NAME) -> str | None:
624
- """? HF Dataset ????????."""
625
  if not HF_REPO or not HF_TOKEN:
626
  return None
627
  try:
@@ -694,7 +694,7 @@ def _start_openviking():
694
 
695
 
696
  def _restore_from_webdav(root: Path, target: str, source: str) -> bool:
697
- """? WebDAV ??????????????????? True?"""
698
  if not WEBDAV_URL or not wd_exists(target):
699
  return False
700
  print(f"[restore] Downloading {target} from WebDAV...")
@@ -719,7 +719,7 @@ def _restore_from_webdav(root: Path, target: str, source: str) -> bool:
719
  p.write_bytes(data)
720
  count += 1
721
  except Exception as e:
722
- print(f"[restore] ? ????????: {rel} -> {e}")
723
  print(f"[restore] Applied {count} incremental file overrides")
724
  except Exception:
725
  print("[restore] No incremental manifest found (clean start)")
@@ -729,7 +729,7 @@ def _restore_from_webdav(root: Path, target: str, source: str) -> bool:
729
 
730
 
731
  def _restore_from_dataset(target: str) -> bool:
732
- """? HF Dataset ???????????? True?"""
733
  print(f"[restore] Trying HF Dataset: {target}...")
734
  path = _hf_download(target)
735
  if not path:
@@ -742,12 +742,12 @@ def _restore_from_dataset(target: str) -> bool:
742
 
743
  def restore(source="auto", filename=None):
744
  """
745
- ??:
746
- - source="auto": WebDAV ?? (?? -> ????) -> HF Dataset ??
747
- - source="webdav": ? WebDAV ???? filename ??
748
- - source="dataset": ? HF Dataset ???? filename ??
749
 
750
- ? filename ? None ?????? openclaw-full.tar.gz?
751
  """
752
  root = Path(STATE_DIR)
753
  root.mkdir(parents=True, exist_ok=True)
@@ -764,13 +764,13 @@ def restore(source="auto", filename=None):
764
  restored = _restore_from_dataset(target)
765
 
766
  if not restored:
767
- print("[restore] No backup found -- fresh start")
768
 
769
  def list_backups() -> dict:
770
- """?? {webdav: [?????], dataset: [?????]}."""
771
  result: dict = {"webdav": [], "dataset": []}
772
 
773
- # WebDAV -- ????????????? banner ??
774
  if _wd_is_configured():
775
  try:
776
  files = wd_list("")
@@ -778,7 +778,7 @@ def list_backups() -> dict:
778
  except Exception as e:
779
  print(f"[backup] WebDAV list error: {e}")
780
 
781
- # HF Dataset -- ??????
782
  if HF_REPO and HF_TOKEN:
783
  try:
784
  backups = _hf_list_backup_files()
@@ -790,12 +790,12 @@ def list_backups() -> dict:
790
  return result
791
 
792
 
793
- # ???????????????????????????????????????????????????????????????????????????????
794
- # ??? (Tick ??) -- ?????????????OV??vectordb???
795
- # ???????????????????????????????????????????????????????????????????????????????
796
 
797
  def _get_hf_file_mtime(filename: str) -> float:
798
- """? HF Dataset ????????????? (unix timestamp)?????? 0"""
799
  try:
800
  from huggingface_hub import HfApi
801
  try:
@@ -807,15 +807,15 @@ def _get_hf_file_mtime(filename: str) -> float:
807
  lc = getattr(entry, 'last_commit', None)
808
  if lc and hasattr(lc, 'date'):
809
  return lc.date.timestamp()
810
- return 0.0 # ?????????????????
811
  return 0.0 # file not found
812
  except Exception as e:
813
- print(f"[backup] ? HF ????: {e}")
814
  return 0.0
815
 
816
 
817
  def _get_wd_last_backup_time() -> float:
818
- """?? WebDAV ???????????? (unix timestamp)?????? 0"""
819
  try:
820
  files = wd_list("")
821
  latest = 0.0
@@ -835,7 +835,7 @@ def _get_wd_last_backup_time() -> float:
835
 
836
 
837
  def _get_hf_large_backup_mtime() -> float:
838
- """?? HF Dataset ????????????? (unix timestamp)????? 0"""
839
  try:
840
  from huggingface_hub import HfApi
841
  try:
@@ -858,28 +858,28 @@ def _get_hf_large_backup_mtime() -> float:
858
 
859
  def _do_one_tick():
860
  """
861
- ?? tick: ???????????? OV -> ?? -> ? OV
862
  """
863
  now = time.time()
864
  today = _today_str()
865
  need_stop_ov = False
866
 
867
- # -- 1. WebDAV --
868
  if not WD_SKIPPED:
869
  last_wd = _get_wd_last_backup_time()
870
  if last_wd == 0.0 or (now - last_wd) >= WEBDAV_INTERVAL * 60:
871
  need_stop_ov = True
872
 
873
- # -- 2. HF ??? --
874
  if not HF_SKIPPED:
875
  last_hf_small = _get_hf_file_mtime(FULL_NAME)
876
  if last_hf_small == 0.0 or (now - last_hf_small) >= HF_SMALL_INTERVAL * 60:
877
  need_stop_ov = True
878
 
879
- # -- 3. HF ??? -- ???????????
880
  hf_big_due = False
881
  if not HF_SKIPPED:
882
- # ??????????????????? OV?
883
  today_file = f"openclaw-full-{_today_str()}.tar.gz"
884
  try:
885
  from huggingface_hub import HfApi
@@ -903,9 +903,9 @@ def _do_one_tick():
903
  need_stop_ov = True
904
 
905
  if not need_stop_ov:
906
- return # ????????
907
 
908
- # -- ? OV -> ?? -> ? OV --
909
  ov_was_running = False
910
  import signal
911
  try:
@@ -926,26 +926,26 @@ def _do_one_tick():
926
  if last_wd == 0.0 or (now - last_wd) >= WEBDAV_INTERVAL * 60:
927
  try:
928
  c = incremental_backup()
929
- print(f"[backup] WebDAV ??: {c} ?????")
930
  full_backup()
931
  except Exception as e:
932
- print(f"[backup] ? WebDAV ?? (??): {e}")
933
 
934
- # HF ???
935
  if not HF_SKIPPED:
936
  last_hf_small = _get_hf_file_mtime(FULL_NAME)
937
  if last_hf_small == 0.0 or (now - last_hf_small) >= HF_SMALL_INTERVAL * 60:
938
  try:
939
  hf_small_backup()
940
  except Exception as e:
941
- print(f"[backup] ? HF????? (??): {e}")
942
 
943
- # HF ?????
944
  if not HF_SKIPPED and hf_big_due:
945
  try:
946
  hf_large_backup()
947
  except Exception as e:
948
- print(f"[backup] ? HF????? (??): {e}")
949
 
950
  if ov_was_running:
951
  _start_openviking()
@@ -953,39 +953,39 @@ def _do_one_tick():
953
 
954
  def scheduler_loop():
955
  """
956
- Tick ???? (??????)??????????????????????
957
- ? OV ?? vectordb ????
958
  """
959
  _ensure_banner()
960
 
961
- print(f"[backup] ?? tick ??")
962
- print(f"[backup] WebDAV ??={WEBDAV_INTERVAL}? ??={WEBDAV_MAX_BACKUPS}?")
963
- print(f"[backup] HF??? ??={HF_SMALL_INTERVAL}?")
964
- print(f"[backup] HF??? ??={HF_BIG_INTERVAL}? ??={DATASET_MAX_BACKUPS}?")
965
 
966
- print(f"[backup] ?? tick ???????5??")
967
 
968
  while True:
969
  time.sleep(5)
970
  _do_one_tick()
971
 
972
- # ???????????????????????????????????????????????????????????????????????????????
973
  # CLI
974
- # ???????????????????????????????????????????????????????????????????????????????
975
 
976
  if __name__ == "__main__":
977
  import argparse
978
- parser = argparse.ArgumentParser(description="WebDAV + HF Dataset ?????/??")
979
  parser.add_argument("cmd", nargs="?", default="restore",
980
  choices=["restore", "list", "incremental", "full", "small", "large", "scheduler", "check"],
981
- help="???? (default: restore)")
982
  parser.add_argument("--source", default="auto",
983
- help="????: auto|webdav|dataset (? restore ??)")
984
  parser.add_argument("--filename", default=None,
985
- help="?????????? (? restore ??)")
986
  args = parser.parse_args()
987
 
988
- # list/check ????? banner????? stdout JSON????????
989
  if args.cmd not in ("list", "check"):
990
  _ensure_banner()
991
 
@@ -996,7 +996,7 @@ if __name__ == "__main__":
996
  print(json.dumps(list_backups(), ensure_ascii=False))
997
  elif args.cmd == "incremental":
998
  c = incremental_backup()
999
- print(f"[backup] WebDAV ??: {c} ?????")
1000
  elif args.cmd == "full":
1001
  full_backup()
1002
  elif args.cmd == "small":
@@ -1016,10 +1016,10 @@ if __name__ == "__main__":
1016
  api = HfApi()
1017
  for entry in api.list_repo_tree(HF_REPO, repo_type="dataset", token=HF_TOKEN, expand=True):
1018
  name = getattr(entry, 'rfilename', '')
1019
- if name == FULL_NAME: # ?????
1020
  lc = getattr(entry, 'last_commit', None)
1021
  if lc and hasattr(lc, 'date'):
1022
- # UTC ????? (UTC+8)
1023
  bj_time = lc.date.astimezone(timezone(timedelta(hours=8)))
1024
  result["last_backup"] = bj_time.strftime("%Y-%m-%d %H:%M:%S")
1025
  if name.startswith("openclaw-full-") and name.endswith(".tar.gz") and name != FULL_NAME:
@@ -1031,8 +1031,8 @@ if __name__ == "__main__":
1031
  result["backup_today"] = True
1032
  result["last_backup"] = bj_time.strftime("%Y-%m-%d %H:%M:%S")
1033
  except Exception as e:
1034
- print(f"[backup] ??????: {e}")
1035
  print(json.dumps(result))
1036
  else:
1037
- print(f"??: {sys.argv[0]} {{restore|incremental|full|small|large|scheduler}}")
1038
  sys.exit(1)
 
1
  #!/usr/bin/env python3
2
  """
3
+ backup-manager.py WebDAV + HF Dataset /
4
+
5
+ :
6
+
7
+
8
+
9
+ WebDAV + BACKUP_WEBDAV_INTERVAL WEBDAV_MAX_BACKUPS
10
+ ( 60 ) ( 30 )
11
+
12
+ HF Dataset BACKUP_HF_INTERVAL_SMALL 1 ()
13
+ ( 30 )
14
+
15
+ HF Dataset BACKUP_HF_INTERVAL_BIG DATASET_MAX_BACKUPS
16
+ ( 30 ) ( 30 )
17
+
18
+
19
+ : HF Dataset 1GB,
20
+ : WebDAV HF Dataset
21
+
22
+ : last_run ,
23
  """
24
 
25
  import hashlib, json, os, tarfile, time, sys
 
28
 
29
  import requests
30
 
31
+ #
32
+ #
33
+ #
34
 
35
  STATE_DIR = os.environ.get("OPENCLAW_STATE_DIR", "/root/.openclaw")
36
 
37
+ # WebDAV
38
  WEBDAV_URL = os.environ.get("WEBDAV_URL", "").rstrip("/")
39
  WEBDAV_USER = os.environ.get("WEBDAV_USERNAME", "")
40
  WEBDAV_PASS = os.environ.get("WEBDAV_PASSWORD", "")
 
42
  WEBDAV_INTERVAL = int(os.environ.get("BACKUP_WEBDAV_INTERVAL", "60"))
43
  WEBDAV_MAX_BACKUPS = int(os.environ.get("WEBDAV_MAX_BACKUPS", "30"))
44
 
45
+ # HF Dataset
46
  HF_REPO = os.environ.get("HF_DATASET", "")
47
  HF_TOKEN = os.environ.get("HF_TOKEN", "")
48
  HF_SMALL_INTERVAL = int(os.environ.get("BACKUP_HF_INTERVAL_SMALL", "30"))
49
  HF_BIG_INTERVAL = int(os.environ.get("BACKUP_HF_INTERVAL_BIG", "30"))
50
  DATASET_MAX_BACKUPS = int(os.environ.get("DATASET_MAX_BACKUPS", "30"))
51
 
52
+ #
53
  FULL_NAME = "openclaw-full.tar.gz"
54
  MANIFEST_NAME = "_incremental_manifest.json"
55
 
56
+ # HF
57
  MAX_HF_STORAGE = 1 * 1024 * 1024 * 1024 # 1 GB
58
 
59
+ #
60
+ HASH_CHUNK_SIZE = 65536 # SHA256
61
+ WD_CHECK_TIMEOUT = 3 # WebDAV
62
+ WD_REQ_TIMEOUT = 30 # WebDAV
63
 
64
+ #
65
  RED = "\033[91m"
66
  GREEN = "\033[92m"
67
  YELLOW = "\033[93m"
68
  RESET = "\033[0m"
69
 
70
+ #
71
+ #
72
+ #
73
 
74
  def _print_banner():
75
+ """"""
76
  global WD_SKIPPED, HF_SKIPPED
77
+ sep = "" * 50
78
+ print(f"[backup] {sep}")
79
 
80
  wd_configured = _wd_is_configured()
81
  if not wd_configured:
82
+ print(f"[backup] WebDAV {RED} : WEBDAV_URL / WEBDAV_USERNAME / WEBDAV_PASSWORD / WEBDAV_BACKUP_DIR {RESET}")
83
  WD_SKIPPED = True
84
  else:
85
+ print(f"[backup] WebDAV {GREEN} {RESET} ={WEBDAV_INTERVAL} ={WEBDAV_MAX_BACKUPS}")
86
  WD_SKIPPED = False
87
 
88
  if not HF_REPO or not HF_TOKEN:
89
  hf_missing = []
90
  if not HF_REPO: hf_missing.append("HF_DATASET")
91
  if not HF_TOKEN: hf_missing.append("HF_TOKEN")
92
+ print(f"[backup] HF {RED} : {', '.join(hf_missing)}{RESET}")
93
+ print(f"[backup] HF {RED} {RESET}")
94
  HF_SKIPPED = True
95
  else:
96
+ print(f"[backup] HF {GREEN} {RESET} ={HF_SMALL_INTERVAL}")
97
+ print(f"[backup] HF {GREEN} {RESET} ={HF_BIG_INTERVAL} ={DATASET_MAX_BACKUPS}")
98
  HF_SKIPPED = False
99
 
100
+ print(f"[backup] {sep}")
101
 
102
+ # banner import
103
  _DISPLAYED_BANNER = False
104
 
105
  def _ensure_banner():
 
108
  _print_banner()
109
  _DISPLAYED_BANNER = True
110
 
111
+ #
112
+ #
113
+ #
114
 
115
  def _create_tar() -> str:
116
+ """ STATE_DIR tar.gz, ."""
117
  tarpath = f"/tmp/openclaw-backup-{int(time.time()*1000)}.tar.gz"
118
  try:
119
  with tarfile.open(tarpath, "w:gz") as tar:
 
127
  try:
128
  tar.add(str(item), arcname=name)
129
  except (PermissionError, FileNotFoundError) as e:
130
+ print(f"[backup] {name}: {e}")
131
  continue
132
  else:
133
+ print(f"[backup] STATE_DIR {STATE_DIR} , ")
134
  root.mkdir(parents=True, exist_ok=True)
135
  size = os.path.getsize(tarpath)
136
  print(f"[backup] Archive created ({size/1024/1024:.1f} MB)")
137
  except (OSError, tarfile.TarError) as e:
138
+ print(f"[backup] : {e}")
139
  if os.path.exists(tarpath):
140
  os.remove(tarpath)
141
+ raise #
142
  return tarpath
143
 
144
  def _file_hash(path: str) -> str:
145
+ """ SHA256 , ."""
146
  try:
147
  h = hashlib.sha256()
148
  with open(path, "rb") as f:
 
153
  h.update(chunk)
154
  return h.hexdigest()
155
  except (OSError, PermissionError, FileNotFoundError) as e:
156
+ print(f"[backup] : {path} {e}")
157
  return ""
158
 
159
  def _today_str() -> str:
160
  return time.strftime("%Y%m%d")
161
 
162
+ #
163
+ # WebDAV HTTP
164
+ #
165
 
166
  def _wd_auth():
167
  return (WEBDAV_USER, WEBDAV_PASS) if WEBDAV_USER else None
 
171
 
172
  def _wd_req(method, path="", **kwargs):
173
  url = _wd_url(path)
174
+ kwargs.setdefault("timeout", WD_REQ_TIMEOUT) # /
175
  resp = requests.request(method, url, auth=_wd_auth(), **kwargs)
176
  resp.raise_for_status()
177
  return resp
178
 
179
  def _wd_check():
180
+ """ WebDAV (3s ), False"""
181
  try:
182
  url = _wd_url("")
183
  resp = requests.request("PROPFIND", url, auth=_wd_auth(), timeout=3)
184
+ resp.raise_for_status() # 401/403/404
185
  return True
186
  except Exception:
187
  return False
 
200
  return _wd_req("GET", path).content
201
 
202
  def wd_delete(path):
203
+ """ WebDAV """
204
  try:
205
  _wd_req("DELETE", path)
206
  return True
207
  except Exception as e:
208
+ print(f"[backup] WebDAV : {path} {e}")
209
  return False
210
 
211
  def wd_mkdir(parts):
212
+ """ WebDAV """
213
  try:
214
  _wd_req("MKCOL", "")
215
  except Exception:
 
222
  pass
223
 
224
  def wd_list(path=""):
225
+ """ WebDAV """
226
  try:
227
  import xml.etree.ElementTree as ET
228
  resp = requests.request(
 
242
  items.append(name)
243
  return [x for x in items if x != WEBDAV_BACKUP_DIR.split("/")[-1]]
244
  except Exception as e:
245
+ print(f"[backup] WebDAV : {e}")
246
  return []
247
 
248
+ #
249
+ # WebDAV (Manifest)
250
+ #
251
 
252
  def _load_manifest() -> dict:
253
  try:
 
260
  wd_upload(MANIFEST_NAME, json.dumps(manifest, indent=2).encode())
261
 
262
  def _scan_and_upload_changes(root: Path, manifest: dict) -> int:
263
+ """ WebDAV. ."""
264
  changed = 0
265
  for fpath in root.rglob("*"):
266
  if not fpath.is_file():
 
292
  changed += 1
293
  except Exception as e:
294
  err_str = str(e)
295
+ # 409 =
296
  if "409" in err_str or "Conflict" in err_str:
297
  manifest[rel] = {
298
  "sha256": cur_h,
 
301
  }
302
  changed += 1
303
  else:
304
+ print(f"[backup] (): {rel} {e}")
305
  return changed
306
 
307
 
308
  def incremental_backup() -> int:
309
+ """, WebDAV. ."""
310
  root = Path(STATE_DIR)
311
  if not root.exists():
312
  return 0
313
 
314
+ # WebDAV
315
  if not _wd_is_configured():
316
+ print(f"[backup] WebDAV , ")
317
  return 0
318
 
319
+ # WebDAV
320
  if not _wd_check():
321
+ print(f"[backup] WebDAV , ")
322
  return 0
323
 
324
  manifest = _load_manifest()
 
328
  try:
329
  _save_manifest(manifest)
330
  except Exception as e:
331
+ print(f"[backup] Manifest : {e}")
332
  return changed
333
 
334
+ #
335
+ # WebDAV +
336
+ #
337
 
338
  def _wd_cleanup():
339
+ """ WEBDAV_MAX_BACKUPS WebDAV """
340
  files = wd_list("")
341
  now = datetime.now()
342
  deleted = 0
 
344
  if not f.startswith("openclaw-full-") or not f.endswith(".tar.gz"):
345
  continue
346
  if f == FULL_NAME:
347
+ continue #
348
  try:
349
  date_str = f.replace("openclaw-full-", "").replace(".tar.gz", "")
350
  dt = datetime.strptime(date_str, "%Y%m%d")
351
  days_old = (now - dt).days
352
  if days_old > WEBDAV_MAX_BACKUPS:
353
  if wd_delete(f):
354
+ print(f"[backup] WebDAV : {f} ({days_old})")
355
  deleted += 1
356
  except ValueError:
357
  continue
358
  if deleted:
359
+ print(f"[backup] WebDAV {deleted} ")
360
  return deleted
361
 
362
  def _wd_is_configured() -> bool:
363
+ """ WebDAV 4 """
364
  return bool(WEBDAV_URL and WEBDAV_USER and WEBDAV_PASS and WEBDAV_BACKUP_DIR)
365
 
366
  def full_backup():
367
+ """ tar WebDAV (: openclaw-full-YYYYMMDD.tar.gz)"""
368
  tarpath = _create_tar()
369
  if _wd_is_configured():
370
+ # WebDAV
371
  if not _wd_check():
372
+ print(f"[backup] WebDAV , ")
373
  os.remove(tarpath)
374
  return
375
  wd_mkdir([])
 
377
  try:
378
  with open(tarpath, "rb") as f:
379
  wd_upload(filename, f.read())
380
+ print(f"[backup] WebDAV {filename}")
381
+ #
382
  with open(tarpath, "rb") as f:
383
  wd_upload(FULL_NAME, f.read())
384
+ #
385
  _wd_cleanup()
386
  except Exception as e:
387
+ print(f"[backup] WebDAV : {e}")
388
  os.remove(tarpath)
389
 
390
+ #
391
+ # HF Dataset
392
+ #
393
 
394
  def _hf_list_backup_files() -> dict | None:
395
  """
396
+ HF Dataset .
397
+ : {: (bytes)}
398
+ {}
399
+ None API
400
  """
401
  if not HF_REPO or not HF_TOKEN:
402
  return {}
 
405
  try:
406
  api = HfApi(timeout=60)
407
  except TypeError:
408
+ api = HfApi() # timeout
409
  backups = {}
410
  for entry in api.list_repo_tree(HF_REPO, repo_type="dataset", token=HF_TOKEN):
411
  name = entry.rfilename
 
413
  backups[name] = entry.size if hasattr(entry, "size") else 0
414
  return backups
415
  except Exception as e:
416
+ print(f"[backup] HF Dataset list API : {e}")
417
+ return None # "" ( {})
418
 
419
  def hf_small_backup():
420
+ """: tar.gz openclaw-full.tar.gz (), """
421
  if not HF_REPO or not HF_TOKEN:
422
  return
423
 
424
  tarpath = _create_tar()
425
  tar_size = os.path.getsize(tarpath)
426
 
427
+ #
428
  backups = _hf_list_backup_files()
429
+ if backups is None: # API
430
+ print(f"[backup] HF: Dataset API ")
431
  os.remove(tarpath)
432
  return
433
  existing_total = sum(backups.values())
434
 
435
  if existing_total + tar_size > MAX_HF_STORAGE:
436
+ print(f"{RED}[backup] HF: 1GB! "
437
+ f"( {existing_total/1024/1024:.1f}MB + "
438
+ f" {tar_size/1024/1024:.1f}MB > 1GB){RESET}")
439
  os.remove(tarpath)
440
  return
441
 
 
444
  try:
445
  api = HfApi(timeout=60)
446
  except TypeError:
447
+ api = HfApi() # timeout
448
  with open(tarpath, "rb") as f:
449
  api.upload_file(
450
  path_or_fileobj=f,
 
453
  repo_type="dataset",
454
  token=HF_TOKEN,
455
  )
456
+ print(f"[backup] HF {FULL_NAME}")
457
  except Exception as e:
458
+ print(f"{RED}[backup] HF: {e}{RESET}")
459
  finally:
460
  if os.path.exists(tarpath):
461
  os.remove(tarpath)
462
 
463
+ #
464
+ # HF Dataset +
465
+ #
466
 
467
  def _hf_cleanup():
468
+ """ DATASET_MAX_BACKUPS HF """
469
  backups = _hf_list_backup_files()
470
+ if not backups: # None(API) {} ()
471
  return 0
472
 
473
  from huggingface_hub import HfApi
 
489
  repo_type="dataset",
490
  token=HF_TOKEN,
491
  )
492
+ print(f"[backup] HF : {name} ({days_old})")
493
  deleted += 1
494
  except (ValueError, Exception):
495
  continue
496
 
497
  if deleted:
498
+ print(f"[backup] HF {deleted} ")
499
  return deleted
500
 
501
  def _hf_make_space(backups: dict, tar_size: int, api) -> bool:
502
+ """, True, False."""
503
+ print(f"[backup] HF "
504
+ f"( {sum(backups.values())/1024/1024:.1f}MB + "
505
+ f" {tar_size/1024/1024:.1f}MB > 1GB)")
506
+ print(f"[backup] ...")
507
 
508
  dated_big = []
509
  for name, sz in backups.items():
 
525
  repo_type="dataset",
526
  token=HF_TOKEN,
527
  )
528
+ print(f"[backup] {name} ( {ds})")
529
  del backups[name]
530
  if sum(backups.values()) + tar_size <= MAX_HF_STORAGE:
531
  return True
532
  except Exception:
533
  continue
534
 
535
+ print(f"{RED}[backup] HF: 1GB "
536
+ f"( {tar_size/1024/1024:.1f}MB > 1GB){RESET}")
537
  return False
538
 
539
 
540
  def _hf_upload_backup(tarpath: str, tar_size: int, filename: str, api) -> None:
541
+ """ HF Dataset."""
542
+ # &
543
  backups = _hf_list_backup_files()
544
+ if backups is None: # API
545
+ print(f"[backup] HF: Dataset API ")
546
  os.remove(tarpath)
547
  return
548
  existing_total = sum(backups.values())
 
551
  if not _hf_make_space(backups, tar_size, api):
552
  os.remove(tarpath)
553
  return
554
+ print(f"[backup] , ")
555
+ backups = _hf_list_backup_files() #
556
  if backups is None:
557
  backups = {}
558
 
559
+ # ,
560
  if filename in backups:
561
  try:
562
  api.delete_file(
 
566
  token=HF_TOKEN,
567
  )
568
  except Exception:
569
+ print(f"[backup] (): {filename}")
570
 
571
+ #
572
  with open(tarpath, "rb") as f:
573
  api.upload_file(
574
  path_or_fileobj=f,
 
577
  repo_type="dataset",
578
  token=HF_TOKEN,
579
  )
580
+ print(f"[backup] HF {filename}")
581
 
582
+ #
583
  _hf_cleanup()
584
 
585
 
586
  def hf_large_backup():
587
  """
588
+ : tar.gz openclaw-full-YYYYMMDD.tar.gz
589
+ ,
590
  """
591
  if not HF_REPO or not HF_TOKEN:
592
  return
 
608
  try:
609
  api = HfApi(timeout=60)
610
  except TypeError:
611
+ api = HfApi() # timeout
612
  _hf_upload_backup(tarpath, tar_size, filename, api)
613
  except Exception as e:
614
+ print(f"{RED}[backup] HF: {e}{RESET}")
615
  finally:
616
  if os.path.exists(tarpath):
617
  os.remove(tarpath)
618
 
619
+ #
620
+ #
621
+ #
622
 
623
  def _hf_download(filename: str = FULL_NAME) -> str | None:
624
+ """ HF Dataset ."""
625
  if not HF_REPO or not HF_TOKEN:
626
  return None
627
  try:
 
694
 
695
 
696
  def _restore_from_webdav(root: Path, target: str, source: str) -> bool:
697
+ """ WebDAV True"""
698
  if not WEBDAV_URL or not wd_exists(target):
699
  return False
700
  print(f"[restore] Downloading {target} from WebDAV...")
 
719
  p.write_bytes(data)
720
  count += 1
721
  except Exception as e:
722
+ print(f"[restore] : {rel} {e}")
723
  print(f"[restore] Applied {count} incremental file overrides")
724
  except Exception:
725
  print("[restore] No incremental manifest found (clean start)")
 
729
 
730
 
731
  def _restore_from_dataset(target: str) -> bool:
732
+ """ HF Dataset True"""
733
  print(f"[restore] Trying HF Dataset: {target}...")
734
  path = _hf_download(target)
735
  if not path:
 
742
 
743
  def restore(source="auto", filename=None):
744
  """
745
+ :
746
+ - source="auto": WebDAV ( ) HF Dataset
747
+ - source="webdav": WebDAV filename
748
+ - source="dataset": HF Dataset filename
749
 
750
+ filename None openclaw-full.tar.gz
751
  """
752
  root = Path(STATE_DIR)
753
  root.mkdir(parents=True, exist_ok=True)
 
764
  restored = _restore_from_dataset(target)
765
 
766
  if not restored:
767
+ print("[restore] No backup found fresh start")
768
 
769
  def list_backups() -> dict:
770
+ """ {webdav: [], dataset: []}."""
771
  result: dict = {"webdav": [], "dataset": []}
772
 
773
+ # WebDAV banner
774
  if _wd_is_configured():
775
  try:
776
  files = wd_list("")
 
778
  except Exception as e:
779
  print(f"[backup] WebDAV list error: {e}")
780
 
781
+ # HF Dataset
782
  if HF_REPO and HF_TOKEN:
783
  try:
784
  backups = _hf_list_backup_files()
 
790
  return result
791
 
792
 
793
+ #
794
+ # (Tick ) OVvectordb
795
+ #
796
 
797
  def _get_hf_file_mtime(filename: str) -> float:
798
+ """ HF Dataset (unix timestamp) 0"""
799
  try:
800
  from huggingface_hub import HfApi
801
  try:
 
807
  lc = getattr(entry, 'last_commit', None)
808
  if lc and hasattr(lc, 'date'):
809
  return lc.date.timestamp()
810
+ return 0.0 #
811
  return 0.0 # file not found
812
  except Exception as e:
813
+ print(f"[backup] HF : {e}")
814
  return 0.0
815
 
816
 
817
  def _get_wd_last_backup_time() -> float:
818
+ """ WebDAV (unix timestamp) 0"""
819
  try:
820
  files = wd_list("")
821
  latest = 0.0
 
835
 
836
 
837
  def _get_hf_large_backup_mtime() -> float:
838
+ """ HF Dataset (unix timestamp) 0"""
839
  try:
840
  from huggingface_hub import HfApi
841
  try:
 
858
 
859
  def _do_one_tick():
860
  """
861
+ tick: OV OV
862
  """
863
  now = time.time()
864
  today = _today_str()
865
  need_stop_ov = False
866
 
867
+ # 1. WebDAV
868
  if not WD_SKIPPED:
869
  last_wd = _get_wd_last_backup_time()
870
  if last_wd == 0.0 or (now - last_wd) >= WEBDAV_INTERVAL * 60:
871
  need_stop_ov = True
872
 
873
+ # 2. HF
874
  if not HF_SKIPPED:
875
  last_hf_small = _get_hf_file_mtime(FULL_NAME)
876
  if last_hf_small == 0.0 or (now - last_hf_small) >= HF_SMALL_INTERVAL * 60:
877
  need_stop_ov = True
878
 
879
+ # 3. HF
880
  hf_big_due = False
881
  if not HF_SKIPPED:
882
+ # OV
883
  today_file = f"openclaw-full-{_today_str()}.tar.gz"
884
  try:
885
  from huggingface_hub import HfApi
 
903
  need_stop_ov = True
904
 
905
  if not need_stop_ov:
906
+ return #
907
 
908
+ # OV OV
909
  ov_was_running = False
910
  import signal
911
  try:
 
926
  if last_wd == 0.0 or (now - last_wd) >= WEBDAV_INTERVAL * 60:
927
  try:
928
  c = incremental_backup()
929
+ print(f"[backup] WebDAV : {c} ")
930
  full_backup()
931
  except Exception as e:
932
+ print(f"[backup] WebDAV (): {e}")
933
 
934
+ # HF
935
  if not HF_SKIPPED:
936
  last_hf_small = _get_hf_file_mtime(FULL_NAME)
937
  if last_hf_small == 0.0 or (now - last_hf_small) >= HF_SMALL_INTERVAL * 60:
938
  try:
939
  hf_small_backup()
940
  except Exception as e:
941
+ print(f"[backup] HF (): {e}")
942
 
943
+ # HF
944
  if not HF_SKIPPED and hf_big_due:
945
  try:
946
  hf_large_backup()
947
  except Exception as e:
948
+ print(f"[backup] HF (): {e}")
949
 
950
  if ov_was_running:
951
  _start_openviking()
 
953
 
954
  def scheduler_loop():
955
  """
956
+ Tick ()
957
+ OV vectordb
958
  """
959
  _ensure_banner()
960
 
961
+ print(f"[backup] tick ")
962
+ print(f"[backup] WebDAV ={WEBDAV_INTERVAL} ={WEBDAV_MAX_BACKUPS}")
963
+ print(f"[backup] HF ={HF_SMALL_INTERVAL}")
964
+ print(f"[backup] HF ={HF_BIG_INTERVAL} ={DATASET_MAX_BACKUPS}")
965
 
966
+ print(f"[backup] tick 5")
967
 
968
  while True:
969
  time.sleep(5)
970
  _do_one_tick()
971
 
972
+ #
973
  # CLI
974
+ #
975
 
976
  if __name__ == "__main__":
977
  import argparse
978
+ parser = argparse.ArgumentParser(description="WebDAV + HF Dataset /")
979
  parser.add_argument("cmd", nargs="?", default="restore",
980
  choices=["restore", "list", "incremental", "full", "small", "large", "scheduler", "check"],
981
+ help=" (default: restore)")
982
  parser.add_argument("--source", default="auto",
983
+ help=": auto|webdav|dataset ( restore )")
984
  parser.add_argument("--filename", default=None,
985
+ help=" ( restore )")
986
  args = parser.parse_args()
987
 
988
+ # list/check banner stdout JSON
989
  if args.cmd not in ("list", "check"):
990
  _ensure_banner()
991
 
 
996
  print(json.dumps(list_backups(), ensure_ascii=False))
997
  elif args.cmd == "incremental":
998
  c = incremental_backup()
999
+ print(f"[backup] WebDAV : {c} ")
1000
  elif args.cmd == "full":
1001
  full_backup()
1002
  elif args.cmd == "small":
 
1016
  api = HfApi()
1017
  for entry in api.list_repo_tree(HF_REPO, repo_type="dataset", token=HF_TOKEN, expand=True):
1018
  name = getattr(entry, 'rfilename', '')
1019
+ if name == FULL_NAME: #
1020
  lc = getattr(entry, 'last_commit', None)
1021
  if lc and hasattr(lc, 'date'):
1022
+ # UTC (UTC+8)
1023
  bj_time = lc.date.astimezone(timezone(timedelta(hours=8)))
1024
  result["last_backup"] = bj_time.strftime("%Y-%m-%d %H:%M:%S")
1025
  if name.startswith("openclaw-full-") and name.endswith(".tar.gz") and name != FULL_NAME:
 
1031
  result["backup_today"] = True
1032
  result["last_backup"] = bj_time.strftime("%Y-%m-%d %H:%M:%S")
1033
  except Exception as e:
1034
+ print(f"[backup] : {e}")
1035
  print(json.dumps(result))
1036
  else:
1037
+ print(f": {sys.argv[0]} {{restore|incremental|full|small|large|scheduler}}")
1038
  sys.exit(1)