Stabilize autosync defaults and startup behavior
Browse files
README.md
CHANGED
|
@@ -64,11 +64,28 @@ Opsional environment variable:
|
|
| 64 |
- `HF_MODEL_FILE` - nama file model di Hub (disarankan: `best.pt`)
|
| 65 |
- `HF_MODEL_REPO_TYPE` - tipe repo HF: `space` (default), `model`, atau `dataset`
|
| 66 |
- `HF_TOKEN` - token untuk upload otomatis saat training selesai
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
Catatan sync otomatis setelah training:
|
| 69 |
- Jika `HF_MODEL_REPO` terisi, aplikasi akan upload `best.pt` ke `HF_MODEL_FILE`.
|
| 70 |
- Jika tidak terisi, training tetap jalan normal dan hanya menyimpan `best.pt` lokal.
|
| 71 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
## Menjalankan Lokal
|
| 73 |
|
| 74 |
```bash
|
|
|
|
| 64 |
- `HF_MODEL_FILE` - nama file model di Hub (disarankan: `best.pt`)
|
| 65 |
- `HF_MODEL_REPO_TYPE` - tipe repo HF: `space` (default), `model`, atau `dataset`
|
| 66 |
- `HF_TOKEN` - token untuk upload otomatis saat training selesai
|
| 67 |
+
- `HF_AUTOSYNC_ENABLED` - aktif/nonaktif auto-sync file repo (`true/false`)
|
| 68 |
+
- `HF_AUTOSYNC_LOCAL_ONLY` - default `true`, artinya autosync hanya jalan dari lokal
|
| 69 |
+
- `HF_AUTOSYNC_REALTIME` - default `false`; jika `true` baru watcher realtime aktif
|
| 70 |
+
- `HF_AUTOSYNC_REPO_ID` - repo target auto-sync (fallback ke `HF_MODEL_REPO`)
|
| 71 |
+
- `HF_AUTOSYNC_REPO_TYPE` - tipe repo target auto-sync (default `space`)
|
| 72 |
+
- `HF_AUTOSYNC_TOKEN` - token auto-sync (fallback ke `HF_TOKEN`)
|
| 73 |
+
- `HF_AUTOSYNC_INTERVAL_SEC` - interval cek perubahan file (default `3` detik)
|
| 74 |
+
- `HF_AUTOSYNC_DEBOUNCE_SEC` - jeda batch commit agar tidak commit tiap perubahan kecil (default `8` detik)
|
| 75 |
+
- `HF_AUTOSYNC_INCLUDE_GLOBS` - pola file yang ingin disync (default `*.py,*.md,requirements.txt,.gitignore`)
|
| 76 |
+
- `HF_AUTOSYNC_MAX_FILE_MB` - batas ukuran file yang di-sync (default `200` MB)
|
| 77 |
+
- `HF_AUTOSYNC_IGNORE_GLOBS` - daftar pola file/folder yang diabaikan (dipisah koma)
|
| 78 |
|
| 79 |
Catatan sync otomatis setelah training:
|
| 80 |
- Jika `HF_MODEL_REPO` terisi, aplikasi akan upload `best.pt` ke `HF_MODEL_FILE`.
|
| 81 |
- Jika tidak terisi, training tetap jalan normal dan hanya menyimpan `best.pt` lokal.
|
| 82 |
|
| 83 |
+
Catatan auto-sync update file repo:
|
| 84 |
+
- Default `HF_AUTOSYNC_ENABLED=false`, jadi tidak ada sync repo saat startup/runtime.
|
| 85 |
+
- Jika ingin sync repo otomatis, aktifkan manual: `HF_AUTOSYNC_ENABLED=true`.
|
| 86 |
+
- Untuk mode realtime, tambahkan `HF_AUTOSYNC_REALTIME=true`.
|
| 87 |
+
- Disarankan tetap fokus ke sync model `best.pt` setelah training (lebih ringan dan stabil).
|
| 88 |
+
|
| 89 |
## Menjalankan Lokal
|
| 90 |
|
| 91 |
```bash
|
app.py
CHANGED
|
@@ -480,12 +480,18 @@ def _parse_globs(raw_value: str) -> list[str]:
|
|
| 480 |
return [g.strip() for g in raw_value.split(",") if g.strip()]
|
| 481 |
|
| 482 |
|
| 483 |
-
def _snapshot_repo_files(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 484 |
snapshot: Dict[str, Tuple[int, int]] = {}
|
| 485 |
for p in root.rglob("*"):
|
| 486 |
if not p.is_file():
|
| 487 |
continue
|
| 488 |
rel = p.relative_to(root).as_posix()
|
|
|
|
|
|
|
| 489 |
if any(fnmatch.fnmatch(rel, pat) for pat in ignore_globs):
|
| 490 |
continue
|
| 491 |
st = p.stat()
|
|
@@ -542,32 +548,50 @@ def _repo_autosync_loop(
|
|
| 542 |
repo_id: str,
|
| 543 |
repo_type: str,
|
| 544 |
interval_sec: int,
|
|
|
|
|
|
|
| 545 |
ignore_globs: list[str],
|
| 546 |
max_file_mb: float,
|
| 547 |
) -> None:
|
| 548 |
-
prev = _snapshot_repo_files(root, ignore_globs)
|
|
|
|
|
|
|
|
|
|
| 549 |
print(
|
| 550 |
-
|
|
|
|
| 551 |
)
|
| 552 |
while True:
|
| 553 |
try:
|
| 554 |
time.sleep(interval_sec)
|
| 555 |
-
curr = _snapshot_repo_files(root, ignore_globs)
|
| 556 |
changed = {rel for rel, sig in curr.items() if prev.get(rel) != sig}
|
| 557 |
deleted = set(prev.keys()) - set(curr.keys())
|
| 558 |
|
| 559 |
if changed or deleted:
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 571 |
except Exception as err:
|
| 572 |
print(f"HF autosync error: {err}")
|
| 573 |
|
|
@@ -575,16 +599,49 @@ def _repo_autosync_loop(
|
|
| 575 |
_repo_autosync_started = False
|
| 576 |
|
| 577 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 578 |
def start_repo_autosync(root: Path) -> str:
|
| 579 |
global _repo_autosync_started
|
| 580 |
if _repo_autosync_started:
|
| 581 |
return "HF autosync sudah aktif."
|
| 582 |
|
| 583 |
repo_id = os.getenv("HF_AUTOSYNC_REPO_ID", "").strip() or os.getenv("HF_MODEL_REPO", "").strip()
|
| 584 |
-
|
| 585 |
-
enabled = _env_true("HF_AUTOSYNC_ENABLED", default=default_enabled)
|
| 586 |
if not enabled:
|
| 587 |
return "HF autosync nonaktif (HF_AUTOSYNC_ENABLED=false)."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 588 |
if not repo_id:
|
| 589 |
return "HF autosync dilewati: set HF_AUTOSYNC_REPO_ID atau HF_MODEL_REPO."
|
| 590 |
|
|
@@ -594,27 +651,61 @@ def start_repo_autosync(root: Path) -> str:
|
|
| 594 |
or "space"
|
| 595 |
)
|
| 596 |
token = os.getenv("HF_AUTOSYNC_TOKEN", "").strip() or os.getenv("HF_TOKEN", "").strip() or None
|
| 597 |
-
interval_sec = max(
|
|
|
|
|
|
|
| 598 |
max_file_mb = max(1.0, float(os.getenv("HF_AUTOSYNC_MAX_FILE_MB", "200").strip() or "200"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 599 |
ignore_globs = _parse_globs(
|
| 600 |
os.getenv(
|
| 601 |
"HF_AUTOSYNC_IGNORE_GLOBS",
|
| 602 |
-
".git/**,__pycache__/**,.venv/**,venv/**,*.pyc,*.pyo,*.tmp",
|
| 603 |
)
|
| 604 |
)
|
| 605 |
|
| 606 |
api = HfApi(token=token)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 607 |
worker = Thread(
|
| 608 |
target=_repo_autosync_loop,
|
| 609 |
-
args=(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 610 |
daemon=True,
|
| 611 |
name="hf-repo-autosync",
|
| 612 |
)
|
| 613 |
worker.start()
|
| 614 |
_repo_autosync_started = True
|
| 615 |
return (
|
| 616 |
-
"HF autosync dinyalakan: "
|
| 617 |
-
f"repo={repo_id}, type={repo_type}, interval={interval_sec}s"
|
| 618 |
)
|
| 619 |
|
| 620 |
|
|
@@ -807,5 +898,8 @@ with gr.Blocks(title="Pengecekan dan Klasifikasi Mahasiswa - YOLOv8") as demo:
|
|
| 807 |
|
| 808 |
|
| 809 |
if __name__ == "__main__":
|
| 810 |
-
|
|
|
|
|
|
|
|
|
|
| 811 |
demo.launch(css=CUSTOM_CSS, ssr_mode=False)
|
|
|
|
| 480 |
return [g.strip() for g in raw_value.split(",") if g.strip()]
|
| 481 |
|
| 482 |
|
| 483 |
+
def _snapshot_repo_files(
|
| 484 |
+
root: Path,
|
| 485 |
+
include_globs: list[str],
|
| 486 |
+
ignore_globs: list[str],
|
| 487 |
+
) -> Dict[str, Tuple[int, int]]:
|
| 488 |
snapshot: Dict[str, Tuple[int, int]] = {}
|
| 489 |
for p in root.rglob("*"):
|
| 490 |
if not p.is_file():
|
| 491 |
continue
|
| 492 |
rel = p.relative_to(root).as_posix()
|
| 493 |
+
if include_globs and not any(fnmatch.fnmatch(rel, pat) for pat in include_globs):
|
| 494 |
+
continue
|
| 495 |
if any(fnmatch.fnmatch(rel, pat) for pat in ignore_globs):
|
| 496 |
continue
|
| 497 |
st = p.stat()
|
|
|
|
| 548 |
repo_id: str,
|
| 549 |
repo_type: str,
|
| 550 |
interval_sec: int,
|
| 551 |
+
debounce_sec: int,
|
| 552 |
+
include_globs: list[str],
|
| 553 |
ignore_globs: list[str],
|
| 554 |
max_file_mb: float,
|
| 555 |
) -> None:
|
| 556 |
+
prev = _snapshot_repo_files(root, include_globs, ignore_globs)
|
| 557 |
+
pending_changed: set[str] = set()
|
| 558 |
+
pending_deleted: set[str] = set()
|
| 559 |
+
pending_since = 0.0
|
| 560 |
print(
|
| 561 |
+
"HF autosync aktif: "
|
| 562 |
+
f"repo={repo_id} type={repo_type} interval={interval_sec}s debounce={debounce_sec}s"
|
| 563 |
)
|
| 564 |
while True:
|
| 565 |
try:
|
| 566 |
time.sleep(interval_sec)
|
| 567 |
+
curr = _snapshot_repo_files(root, include_globs, ignore_globs)
|
| 568 |
changed = {rel for rel, sig in curr.items() if prev.get(rel) != sig}
|
| 569 |
deleted = set(prev.keys()) - set(curr.keys())
|
| 570 |
|
| 571 |
if changed or deleted:
|
| 572 |
+
if pending_since <= 0:
|
| 573 |
+
pending_since = time.time()
|
| 574 |
+
pending_changed |= changed
|
| 575 |
+
pending_deleted |= deleted
|
| 576 |
+
pending_changed -= pending_deleted
|
| 577 |
+
|
| 578 |
+
if pending_changed or pending_deleted:
|
| 579 |
+
elapsed = time.time() - pending_since if pending_since > 0 else 0.0
|
| 580 |
+
if elapsed >= debounce_sec:
|
| 581 |
+
msg = _commit_repo_deltas(
|
| 582 |
+
api=api,
|
| 583 |
+
root=root,
|
| 584 |
+
repo_id=repo_id,
|
| 585 |
+
repo_type=repo_type,
|
| 586 |
+
changed=pending_changed,
|
| 587 |
+
deleted=pending_deleted,
|
| 588 |
+
max_file_mb=max_file_mb,
|
| 589 |
+
)
|
| 590 |
+
print(msg)
|
| 591 |
+
prev = curr
|
| 592 |
+
pending_changed = set()
|
| 593 |
+
pending_deleted = set()
|
| 594 |
+
pending_since = 0.0
|
| 595 |
except Exception as err:
|
| 596 |
print(f"HF autosync error: {err}")
|
| 597 |
|
|
|
|
| 599 |
_repo_autosync_started = False
|
| 600 |
|
| 601 |
|
| 602 |
+
def _running_in_hf_space() -> bool:
|
| 603 |
+
return bool(os.getenv("SPACE_ID", "").strip())
|
| 604 |
+
|
| 605 |
+
|
| 606 |
+
def _sync_repo_once(
|
| 607 |
+
root: Path,
|
| 608 |
+
api: HfApi,
|
| 609 |
+
repo_id: str,
|
| 610 |
+
repo_type: str,
|
| 611 |
+
include_globs: list[str],
|
| 612 |
+
ignore_globs: list[str],
|
| 613 |
+
max_file_mb: float,
|
| 614 |
+
) -> str:
|
| 615 |
+
current = _snapshot_repo_files(root, include_globs, ignore_globs)
|
| 616 |
+
changed = set(current.keys())
|
| 617 |
+
if not changed:
|
| 618 |
+
return "HF build-sync dilewati: tidak ada file yang cocok pola include."
|
| 619 |
+
msg = _commit_repo_deltas(
|
| 620 |
+
api=api,
|
| 621 |
+
root=root,
|
| 622 |
+
repo_id=repo_id,
|
| 623 |
+
repo_type=repo_type,
|
| 624 |
+
changed=changed,
|
| 625 |
+
deleted=set(),
|
| 626 |
+
max_file_mb=max_file_mb,
|
| 627 |
+
)
|
| 628 |
+
return f"HF build-sync (one-shot): {msg}"
|
| 629 |
+
|
| 630 |
+
|
| 631 |
def start_repo_autosync(root: Path) -> str:
|
| 632 |
global _repo_autosync_started
|
| 633 |
if _repo_autosync_started:
|
| 634 |
return "HF autosync sudah aktif."
|
| 635 |
|
| 636 |
repo_id = os.getenv("HF_AUTOSYNC_REPO_ID", "").strip() or os.getenv("HF_MODEL_REPO", "").strip()
|
| 637 |
+
enabled = _env_true("HF_AUTOSYNC_ENABLED", default=False)
|
|
|
|
| 638 |
if not enabled:
|
| 639 |
return "HF autosync nonaktif (HF_AUTOSYNC_ENABLED=false)."
|
| 640 |
+
|
| 641 |
+
local_only = _env_true("HF_AUTOSYNC_LOCAL_ONLY", default=True)
|
| 642 |
+
if local_only and _running_in_hf_space():
|
| 643 |
+
return "HF autosync dilewati di HF Space (mode local-only aktif)."
|
| 644 |
+
|
| 645 |
if not repo_id:
|
| 646 |
return "HF autosync dilewati: set HF_AUTOSYNC_REPO_ID atau HF_MODEL_REPO."
|
| 647 |
|
|
|
|
| 651 |
or "space"
|
| 652 |
)
|
| 653 |
token = os.getenv("HF_AUTOSYNC_TOKEN", "").strip() or os.getenv("HF_TOKEN", "").strip() or None
|
| 654 |
+
interval_sec = max(2, int(float(os.getenv("HF_AUTOSYNC_INTERVAL_SEC", "3").strip() or "3")))
|
| 655 |
+
debounce_sec = max(2, int(float(os.getenv("HF_AUTOSYNC_DEBOUNCE_SEC", "8").strip() or "8")))
|
| 656 |
+
realtime = _env_true("HF_AUTOSYNC_REALTIME", default=False)
|
| 657 |
max_file_mb = max(1.0, float(os.getenv("HF_AUTOSYNC_MAX_FILE_MB", "200").strip() or "200"))
|
| 658 |
+
include_globs = _parse_globs(
|
| 659 |
+
os.getenv(
|
| 660 |
+
"HF_AUTOSYNC_INCLUDE_GLOBS",
|
| 661 |
+
"*.py,*.md,requirements.txt,.gitignore",
|
| 662 |
+
)
|
| 663 |
+
)
|
| 664 |
ignore_globs = _parse_globs(
|
| 665 |
os.getenv(
|
| 666 |
"HF_AUTOSYNC_IGNORE_GLOBS",
|
| 667 |
+
".git/**,__pycache__/**,.venv/**,venv/**,*.pyc,*.pyo,*.tmp,runs_train/**,prepared_dataset/**",
|
| 668 |
)
|
| 669 |
)
|
| 670 |
|
| 671 |
api = HfApi(token=token)
|
| 672 |
+
if not realtime:
|
| 673 |
+
try:
|
| 674 |
+
_repo_autosync_started = True
|
| 675 |
+
return _sync_repo_once(
|
| 676 |
+
root=root,
|
| 677 |
+
api=api,
|
| 678 |
+
repo_id=repo_id,
|
| 679 |
+
repo_type=repo_type,
|
| 680 |
+
include_globs=include_globs,
|
| 681 |
+
ignore_globs=ignore_globs,
|
| 682 |
+
max_file_mb=max_file_mb,
|
| 683 |
+
)
|
| 684 |
+
except Exception as err:
|
| 685 |
+
_repo_autosync_started = False
|
| 686 |
+
return f"HF build-sync gagal (one-shot): {err}"
|
| 687 |
+
|
| 688 |
worker = Thread(
|
| 689 |
target=_repo_autosync_loop,
|
| 690 |
+
args=(
|
| 691 |
+
root,
|
| 692 |
+
api,
|
| 693 |
+
repo_id,
|
| 694 |
+
repo_type,
|
| 695 |
+
interval_sec,
|
| 696 |
+
debounce_sec,
|
| 697 |
+
include_globs,
|
| 698 |
+
ignore_globs,
|
| 699 |
+
max_file_mb,
|
| 700 |
+
),
|
| 701 |
daemon=True,
|
| 702 |
name="hf-repo-autosync",
|
| 703 |
)
|
| 704 |
worker.start()
|
| 705 |
_repo_autosync_started = True
|
| 706 |
return (
|
| 707 |
+
"HF autosync realtime dinyalakan: "
|
| 708 |
+
f"repo={repo_id}, type={repo_type}, interval={interval_sec}s, debounce={debounce_sec}s"
|
| 709 |
)
|
| 710 |
|
| 711 |
|
|
|
|
| 898 |
|
| 899 |
|
| 900 |
if __name__ == "__main__":
|
| 901 |
+
if _env_true("HF_AUTOSYNC_ENABLED", default=False):
|
| 902 |
+
print(start_repo_autosync(Path(__file__).resolve().parent))
|
| 903 |
+
else:
|
| 904 |
+
print("HF autosync nonaktif (HF_AUTOSYNC_ENABLED=false).")
|
| 905 |
demo.launch(css=CUSTOM_CSS, ssr_mode=False)
|