"""Seed-time org attribution for malformed-org canonical ids. A draft id with no `/` whose leading token is a known developer org couldn't have its org parsed — the org delimiter is a `-`/`.` instead of `/` — so it lands with a null org_id, orphaned from its developer (`cohere-march-2024`, `deepseek-coder`). This pass attributes the org from that leading token. org_id is DECOUPLED from the id string (the registry already maps `Qwen/…` → org `alibaba`), so attribution is safe even when the org token also appears in the model NAME (`deepseek-coder` → org `deepseek`, id kept verbatim). It does NOT touch an entry that is actually a DUPLICATE of an existing real repo of the same developer (`deepseek-v2-lite-chat` ≡ `deepseek-ai/DeepSeek-V2-Lite-Chat`, `nvidia.nemotron-nano-9b-v2` ≡ `nvidia/NVIDIA-Nemotron-Nano-9B-v2`). Those don't normalize-collide (the real repo's name repeats the org), and org-tagging them in isolation would collide on the humanized display name — they need a fold into the real repo, which is a separate concern. Detection: same developer + same org-token-stripped model name as a more-canonical (slashed) entry. """ import re from collections import defaultdict _LEAD_RE = re.compile(r"([A-Za-z0-9]+)([-.])(.+)") _PLACEHOLDER_ORGS = frozenset({"unknown", "none", "na", ""}) _SEP = re.compile(r"[-_.\s]+") def _dev_of(e: dict, hf_to_dev: dict) -> str | None: org = e.get("org_id") if org and str(org).lower() not in _PLACEHOLDER_ORGS: return str(org).lower() cid = str(e.get("id", "")) if "/" in cid: return hf_to_dev.get(cid.split("/", 1)[0].lower()) m = _LEAD_RE.match(cid) return hf_to_dev.get(m.group(1).lower()) if m else None def _name_key(cid: str, dev: str | None) -> str: """Model name normalized after stripping a leading developer token, so `deepseek-ai/DeepSeek-V2-Lite-Chat` and `deepseek-v2-lite-chat` share a key.""" name = cid.split("/", 1)[1] if "/" in cid else cid nl = name.lower() if dev: for tok in (dev, dev.replace("-", ""), dev.split("-")[0]): if tok and nl.startswith(tok): nl = nl[len(tok):] break return _SEP.sub("", nl).strip("-_. ") def attribute_orgs(entries: list[dict], hf_to_dev: dict[str, str]): """For each malformed-org draft id whose leading token is a known org: - if it DUPLICATES an existing real repo of that org (same developer + same org-token-stripped model name), return it in the merge map {draft -> real repo} so the fold collapses it into the real repo; - otherwise attribute org_id (standalone draft) in place. Returns (entries, merge_map). The merge map carries the cross-spelling duplicates the resolver's normalize missed (it's org-blind: `deepseek` vs `deepseek-ai`, the org repeated in the model name, letter-dot not split).""" seen: dict[tuple, list[str]] = defaultdict(list) for e in entries: dev = _dev_of(e, hf_to_dev) if dev: seen[(dev, _name_key(str(e.get("id", "")), dev))].append(str(e.get("id", ""))) merges: dict[str, str] = {} for e in entries: cid = str(e.get("id", "")) if "/" in cid: continue org = e.get("org_id") if org and str(org).lower() not in _PLACEHOLDER_ORGS: continue m = _LEAD_RE.match(cid) if not m: continue dev = hf_to_dev.get(m.group(1).lower()) if not dev: continue twins = [i for i in seen.get((dev, _name_key(cid, dev)), []) if i != cid and "/" in i] if twins: merges[cid] = sorted(twins)[0] # fold the draft into the real repo else: e["org_id"] = dev # standalone draft: just attribute return entries, merges