firgiawann commited on
Commit
df511f5
·
verified ·
1 Parent(s): 445417a

Stabilize autosync defaults and startup behavior

Browse files
Files changed (2) hide show
  1. README.md +17 -0
  2. app.py +117 -23
README.md CHANGED
@@ -64,11 +64,28 @@ Opsional environment variable:
64
  - `HF_MODEL_FILE` - nama file model di Hub (disarankan: `best.pt`)
65
  - `HF_MODEL_REPO_TYPE` - tipe repo HF: `space` (default), `model`, atau `dataset`
66
  - `HF_TOKEN` - token untuk upload otomatis saat training selesai
 
 
 
 
 
 
 
 
 
 
 
67
 
68
  Catatan sync otomatis setelah training:
69
  - Jika `HF_MODEL_REPO` terisi, aplikasi akan upload `best.pt` ke `HF_MODEL_FILE`.
70
  - Jika tidak terisi, training tetap jalan normal dan hanya menyimpan `best.pt` lokal.
71
 
 
 
 
 
 
 
72
  ## Menjalankan Lokal
73
 
74
  ```bash
 
64
  - `HF_MODEL_FILE` - nama file model di Hub (disarankan: `best.pt`)
65
  - `HF_MODEL_REPO_TYPE` - tipe repo HF: `space` (default), `model`, atau `dataset`
66
  - `HF_TOKEN` - token untuk upload otomatis saat training selesai
67
+ - `HF_AUTOSYNC_ENABLED` - aktif/nonaktif auto-sync file repo (`true/false`)
68
+ - `HF_AUTOSYNC_LOCAL_ONLY` - default `true`, artinya autosync hanya jalan dari lokal
69
+ - `HF_AUTOSYNC_REALTIME` - default `false`; jika `true` baru watcher realtime aktif
70
+ - `HF_AUTOSYNC_REPO_ID` - repo target auto-sync (fallback ke `HF_MODEL_REPO`)
71
+ - `HF_AUTOSYNC_REPO_TYPE` - tipe repo target auto-sync (default `space`)
72
+ - `HF_AUTOSYNC_TOKEN` - token auto-sync (fallback ke `HF_TOKEN`)
73
+ - `HF_AUTOSYNC_INTERVAL_SEC` - interval cek perubahan file (default `3` detik)
74
+ - `HF_AUTOSYNC_DEBOUNCE_SEC` - jeda batch commit agar tidak commit tiap perubahan kecil (default `8` detik)
75
+ - `HF_AUTOSYNC_INCLUDE_GLOBS` - pola file yang ingin disync (default `*.py,*.md,requirements.txt,.gitignore`)
76
+ - `HF_AUTOSYNC_MAX_FILE_MB` - batas ukuran file yang di-sync (default `200` MB)
77
+ - `HF_AUTOSYNC_IGNORE_GLOBS` - daftar pola file/folder yang diabaikan (dipisah koma)
78
 
79
  Catatan sync otomatis setelah training:
80
  - Jika `HF_MODEL_REPO` terisi, aplikasi akan upload `best.pt` ke `HF_MODEL_FILE`.
81
  - Jika tidak terisi, training tetap jalan normal dan hanya menyimpan `best.pt` lokal.
82
 
83
+ Catatan auto-sync update file repo:
84
+ - Default `HF_AUTOSYNC_ENABLED=false`, jadi tidak ada sync repo saat startup/runtime.
85
+ - Jika ingin sync repo otomatis, aktifkan manual: `HF_AUTOSYNC_ENABLED=true`.
86
+ - Untuk mode realtime, tambahkan `HF_AUTOSYNC_REALTIME=true`.
87
+ - Disarankan tetap fokus ke sync model `best.pt` setelah training (lebih ringan dan stabil).
88
+
89
  ## Menjalankan Lokal
90
 
91
  ```bash
app.py CHANGED
@@ -480,12 +480,18 @@ def _parse_globs(raw_value: str) -> list[str]:
480
  return [g.strip() for g in raw_value.split(",") if g.strip()]
481
 
482
 
483
- def _snapshot_repo_files(root: Path, ignore_globs: list[str]) -> Dict[str, Tuple[int, int]]:
 
 
 
 
484
  snapshot: Dict[str, Tuple[int, int]] = {}
485
  for p in root.rglob("*"):
486
  if not p.is_file():
487
  continue
488
  rel = p.relative_to(root).as_posix()
 
 
489
  if any(fnmatch.fnmatch(rel, pat) for pat in ignore_globs):
490
  continue
491
  st = p.stat()
@@ -542,32 +548,50 @@ def _repo_autosync_loop(
542
  repo_id: str,
543
  repo_type: str,
544
  interval_sec: int,
 
 
545
  ignore_globs: list[str],
546
  max_file_mb: float,
547
  ) -> None:
548
- prev = _snapshot_repo_files(root, ignore_globs)
 
 
 
549
  print(
550
- f"HF autosync aktif: repo={repo_id} type={repo_type} interval={interval_sec}s"
 
551
  )
552
  while True:
553
  try:
554
  time.sleep(interval_sec)
555
- curr = _snapshot_repo_files(root, ignore_globs)
556
  changed = {rel for rel, sig in curr.items() if prev.get(rel) != sig}
557
  deleted = set(prev.keys()) - set(curr.keys())
558
 
559
  if changed or deleted:
560
- msg = _commit_repo_deltas(
561
- api=api,
562
- root=root,
563
- repo_id=repo_id,
564
- repo_type=repo_type,
565
- changed=changed,
566
- deleted=deleted,
567
- max_file_mb=max_file_mb,
568
- )
569
- print(msg)
570
- prev = curr
 
 
 
 
 
 
 
 
 
 
 
 
571
  except Exception as err:
572
  print(f"HF autosync error: {err}")
573
 
@@ -575,16 +599,49 @@ def _repo_autosync_loop(
575
  _repo_autosync_started = False
576
 
577
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
578
  def start_repo_autosync(root: Path) -> str:
579
  global _repo_autosync_started
580
  if _repo_autosync_started:
581
  return "HF autosync sudah aktif."
582
 
583
  repo_id = os.getenv("HF_AUTOSYNC_REPO_ID", "").strip() or os.getenv("HF_MODEL_REPO", "").strip()
584
- default_enabled = bool(repo_id)
585
- enabled = _env_true("HF_AUTOSYNC_ENABLED", default=default_enabled)
586
  if not enabled:
587
  return "HF autosync nonaktif (HF_AUTOSYNC_ENABLED=false)."
 
 
 
 
 
588
  if not repo_id:
589
  return "HF autosync dilewati: set HF_AUTOSYNC_REPO_ID atau HF_MODEL_REPO."
590
 
@@ -594,27 +651,61 @@ def start_repo_autosync(root: Path) -> str:
594
  or "space"
595
  )
596
  token = os.getenv("HF_AUTOSYNC_TOKEN", "").strip() or os.getenv("HF_TOKEN", "").strip() or None
597
- interval_sec = max(3, int(float(os.getenv("HF_AUTOSYNC_INTERVAL_SEC", "15").strip() or "15")))
 
 
598
  max_file_mb = max(1.0, float(os.getenv("HF_AUTOSYNC_MAX_FILE_MB", "200").strip() or "200"))
 
 
 
 
 
 
599
  ignore_globs = _parse_globs(
600
  os.getenv(
601
  "HF_AUTOSYNC_IGNORE_GLOBS",
602
- ".git/**,__pycache__/**,.venv/**,venv/**,*.pyc,*.pyo,*.tmp",
603
  )
604
  )
605
 
606
  api = HfApi(token=token)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
607
  worker = Thread(
608
  target=_repo_autosync_loop,
609
- args=(root, api, repo_id, repo_type, interval_sec, ignore_globs, max_file_mb),
 
 
 
 
 
 
 
 
 
 
610
  daemon=True,
611
  name="hf-repo-autosync",
612
  )
613
  worker.start()
614
  _repo_autosync_started = True
615
  return (
616
- "HF autosync dinyalakan: "
617
- f"repo={repo_id}, type={repo_type}, interval={interval_sec}s"
618
  )
619
 
620
 
@@ -807,5 +898,8 @@ with gr.Blocks(title="Pengecekan dan Klasifikasi Mahasiswa - YOLOv8") as demo:
807
 
808
 
809
  if __name__ == "__main__":
810
- print(start_repo_autosync(Path(__file__).resolve().parent))
 
 
 
811
  demo.launch(css=CUSTOM_CSS, ssr_mode=False)
 
480
  return [g.strip() for g in raw_value.split(",") if g.strip()]
481
 
482
 
483
+ def _snapshot_repo_files(
484
+ root: Path,
485
+ include_globs: list[str],
486
+ ignore_globs: list[str],
487
+ ) -> Dict[str, Tuple[int, int]]:
488
  snapshot: Dict[str, Tuple[int, int]] = {}
489
  for p in root.rglob("*"):
490
  if not p.is_file():
491
  continue
492
  rel = p.relative_to(root).as_posix()
493
+ if include_globs and not any(fnmatch.fnmatch(rel, pat) for pat in include_globs):
494
+ continue
495
  if any(fnmatch.fnmatch(rel, pat) for pat in ignore_globs):
496
  continue
497
  st = p.stat()
 
548
  repo_id: str,
549
  repo_type: str,
550
  interval_sec: int,
551
+ debounce_sec: int,
552
+ include_globs: list[str],
553
  ignore_globs: list[str],
554
  max_file_mb: float,
555
  ) -> None:
556
+ prev = _snapshot_repo_files(root, include_globs, ignore_globs)
557
+ pending_changed: set[str] = set()
558
+ pending_deleted: set[str] = set()
559
+ pending_since = 0.0
560
  print(
561
+ "HF autosync aktif: "
562
+ f"repo={repo_id} type={repo_type} interval={interval_sec}s debounce={debounce_sec}s"
563
  )
564
  while True:
565
  try:
566
  time.sleep(interval_sec)
567
+ curr = _snapshot_repo_files(root, include_globs, ignore_globs)
568
  changed = {rel for rel, sig in curr.items() if prev.get(rel) != sig}
569
  deleted = set(prev.keys()) - set(curr.keys())
570
 
571
  if changed or deleted:
572
+ if pending_since <= 0:
573
+ pending_since = time.time()
574
+ pending_changed |= changed
575
+ pending_deleted |= deleted
576
+ pending_changed -= pending_deleted
577
+
578
+ if pending_changed or pending_deleted:
579
+ elapsed = time.time() - pending_since if pending_since > 0 else 0.0
580
+ if elapsed >= debounce_sec:
581
+ msg = _commit_repo_deltas(
582
+ api=api,
583
+ root=root,
584
+ repo_id=repo_id,
585
+ repo_type=repo_type,
586
+ changed=pending_changed,
587
+ deleted=pending_deleted,
588
+ max_file_mb=max_file_mb,
589
+ )
590
+ print(msg)
591
+ prev = curr
592
+ pending_changed = set()
593
+ pending_deleted = set()
594
+ pending_since = 0.0
595
  except Exception as err:
596
  print(f"HF autosync error: {err}")
597
 
 
599
  _repo_autosync_started = False
600
 
601
 
602
+ def _running_in_hf_space() -> bool:
603
+ return bool(os.getenv("SPACE_ID", "").strip())
604
+
605
+
606
+ def _sync_repo_once(
607
+ root: Path,
608
+ api: HfApi,
609
+ repo_id: str,
610
+ repo_type: str,
611
+ include_globs: list[str],
612
+ ignore_globs: list[str],
613
+ max_file_mb: float,
614
+ ) -> str:
615
+ current = _snapshot_repo_files(root, include_globs, ignore_globs)
616
+ changed = set(current.keys())
617
+ if not changed:
618
+ return "HF build-sync dilewati: tidak ada file yang cocok pola include."
619
+ msg = _commit_repo_deltas(
620
+ api=api,
621
+ root=root,
622
+ repo_id=repo_id,
623
+ repo_type=repo_type,
624
+ changed=changed,
625
+ deleted=set(),
626
+ max_file_mb=max_file_mb,
627
+ )
628
+ return f"HF build-sync (one-shot): {msg}"
629
+
630
+
631
  def start_repo_autosync(root: Path) -> str:
632
  global _repo_autosync_started
633
  if _repo_autosync_started:
634
  return "HF autosync sudah aktif."
635
 
636
  repo_id = os.getenv("HF_AUTOSYNC_REPO_ID", "").strip() or os.getenv("HF_MODEL_REPO", "").strip()
637
+ enabled = _env_true("HF_AUTOSYNC_ENABLED", default=False)
 
638
  if not enabled:
639
  return "HF autosync nonaktif (HF_AUTOSYNC_ENABLED=false)."
640
+
641
+ local_only = _env_true("HF_AUTOSYNC_LOCAL_ONLY", default=True)
642
+ if local_only and _running_in_hf_space():
643
+ return "HF autosync dilewati di HF Space (mode local-only aktif)."
644
+
645
  if not repo_id:
646
  return "HF autosync dilewati: set HF_AUTOSYNC_REPO_ID atau HF_MODEL_REPO."
647
 
 
651
  or "space"
652
  )
653
  token = os.getenv("HF_AUTOSYNC_TOKEN", "").strip() or os.getenv("HF_TOKEN", "").strip() or None
654
+ interval_sec = max(2, int(float(os.getenv("HF_AUTOSYNC_INTERVAL_SEC", "3").strip() or "3")))
655
+ debounce_sec = max(2, int(float(os.getenv("HF_AUTOSYNC_DEBOUNCE_SEC", "8").strip() or "8")))
656
+ realtime = _env_true("HF_AUTOSYNC_REALTIME", default=False)
657
  max_file_mb = max(1.0, float(os.getenv("HF_AUTOSYNC_MAX_FILE_MB", "200").strip() or "200"))
658
+ include_globs = _parse_globs(
659
+ os.getenv(
660
+ "HF_AUTOSYNC_INCLUDE_GLOBS",
661
+ "*.py,*.md,requirements.txt,.gitignore",
662
+ )
663
+ )
664
  ignore_globs = _parse_globs(
665
  os.getenv(
666
  "HF_AUTOSYNC_IGNORE_GLOBS",
667
+ ".git/**,__pycache__/**,.venv/**,venv/**,*.pyc,*.pyo,*.tmp,runs_train/**,prepared_dataset/**",
668
  )
669
  )
670
 
671
  api = HfApi(token=token)
672
+ if not realtime:
673
+ try:
674
+ _repo_autosync_started = True
675
+ return _sync_repo_once(
676
+ root=root,
677
+ api=api,
678
+ repo_id=repo_id,
679
+ repo_type=repo_type,
680
+ include_globs=include_globs,
681
+ ignore_globs=ignore_globs,
682
+ max_file_mb=max_file_mb,
683
+ )
684
+ except Exception as err:
685
+ _repo_autosync_started = False
686
+ return f"HF build-sync gagal (one-shot): {err}"
687
+
688
  worker = Thread(
689
  target=_repo_autosync_loop,
690
+ args=(
691
+ root,
692
+ api,
693
+ repo_id,
694
+ repo_type,
695
+ interval_sec,
696
+ debounce_sec,
697
+ include_globs,
698
+ ignore_globs,
699
+ max_file_mb,
700
+ ),
701
  daemon=True,
702
  name="hf-repo-autosync",
703
  )
704
  worker.start()
705
  _repo_autosync_started = True
706
  return (
707
+ "HF autosync realtime dinyalakan: "
708
+ f"repo={repo_id}, type={repo_type}, interval={interval_sec}s, debounce={debounce_sec}s"
709
  )
710
 
711
 
 
898
 
899
 
900
  if __name__ == "__main__":
901
+ if _env_true("HF_AUTOSYNC_ENABLED", default=False):
902
+ print(start_repo_autosync(Path(__file__).resolve().parent))
903
+ else:
904
+ print("HF autosync nonaktif (HF_AUTOSYNC_ENABLED=false).")
905
  demo.launch(css=CUSTOM_CSS, ssr_mode=False)