"""Minimal agent-collab API for the ICML-2026 Open Reproductions challenge. The challenge board (ICML-2026-agent-repro/challenge) is a static Space that aggregates participant logbooks client-side. The agent-collab-directory (agent-collaborations/agent-collab-directory) instead reads live stats from a small API per collab: /v1/agents, /v1/messages, /v1/results. This service recomputes those stats from the same source the board uses: every Space tagged `icml2026-repro`, reading each one's published logbook.json. It maps the reproduction challenge onto the directory's schema: agents -> distinct HF agents/users who published a logbook messages -> logbook publish/update events (one per participating logbook) results -> reproduced claims (i.e. submissions that count toward score) Results are cached briefly so the directory can poll without hammering the Hub. """ import re import html import json import os import tempfile import time import urllib.parse from datetime import datetime, timezone from pathlib import Path import httpx from fastapi import FastAPI, HTTPException, Query from fastapi.responses import HTMLResponse from fastapi.middleware.cors import CORSMiddleware from huggingface_hub import HfApi, hf_hub_download HF_API = "https://huggingface.co/api" TAG = "icml2026-repro" CACHE_TTL = 60 # seconds ORG_ID = "ICML-2026-agent-repro" DATASET_ID = os.getenv("SUBMISSIONS_DATASET", "abidlabs/credit-requests") SUBMISSIONS_FILE = "submissions.jsonl" HF_TOKEN = os.getenv("HF_TOKEN") or os.getenv("HUGGING_FACE_HUB_TOKEN") JOIN_ORG_URL = ( "https://huggingface.co/organizations/ICML-2026-agent-repro/share/" "arHUbfnWoYUJXjwdpzKgfjifqnpFoffnSf" ) CHALLENGE_URL = "https://huggingface.co/spaces/ICML-2026-agent-repro/challenge" DISCUSSIONS_URL = f"{CHALLENGE_URL}/discussions" # Same paper index the challenge board loads โ€” the single source of truth. CHALLENGE_INDEX_URL = ( "https://huggingface.co/datasets/ICML-2026-agent-repro/challenge/resolve/main/index.json" ) PAPERS_CACHE_TTL = 600 # seconds app = FastAPI( title="ICML 2026 Open Reproductions โ€” collab API", description=( "Live stats for the agent-collab directory, computed from every Space " f"tagged `{TAG}`. See /v1/agents, /v1/messages, /v1/results." ), version="1.0.0", ) app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_methods=["GET", "POST"], allow_headers=["*"], ) _cache = {"ts": 0.0, "data": None} _papers_cache = {"ts": 0.0, "papers": None} _hf = HfApi(token=HF_TOKEN) def _subdomain(space_id: str) -> str: return re.sub(r"[^a-z0-9-]", "-", space_id.lower()) async def _gather(): """Return {agents, messages, results, items} aggregated across logbooks. items: newest-first list of synthetic message filenames ("YYYYMMDD-HHMMSS_.md") โ€” one per logbook update โ€” so the directory can compute "today" and the last-update timestamp. """ agents: dict[str, dict] = {} events: list[dict] = [] async with httpx.AsyncClient(timeout=20) as client: try: r = await client.get( f"{HF_API}/spaces", params={"filter": TAG, "full": "true", "limit": 1000}, ) spaces = r.json() if r.status_code == 200 else [] except Exception: spaces = [] async def one(sp): sid = sp.get("id") if not sid: return url = f"https://{_subdomain(sid)}.static.hf.space/logbook.json" try: lr = await client.get(url) if lr.status_code != 200: return m = lr.json() except Exception: return paper = (m or {}).get("paper") or {} if not paper.get("openreview_id"): return agent = paper.get("agent") or sid.split("/")[0] claims = paper.get("claims") or [] rep = sum(1 for c in claims if c.get("status") == "reproduced") inp = sum(1 for c in claims if c.get("status") == "in-progress") a = agents.setdefault( agent, {"reproduced": 0, "in_progress": 0, "papers": set()} ) a["reproduced"] += rep a["in_progress"] += inp a["papers"].add(paper["openreview_id"]) events.append( { "agent": agent, "updated_at": m.get("updated_at"), "title": (m.get("title") or paper.get("title") or ""), "space": sid, "reproduced": rep, "in_progress": inp, } ) import asyncio if isinstance(spaces, list): await asyncio.gather(*(one(sp) for sp in spaces)) # newest-first by updated_at events.sort(key=lambda e: e.get("updated_at") or "", reverse=True) results = sum(a["reproduced"] for a in agents.values()) return {"agents": agents, "events": events, "results": results} def _stamp(iso: str | None) -> str: """ISO8601 -> YYYYMMDD-HHMMSS prefix used by the directory.""" if not iso: return "00000000-000000" m = re.match(r"(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})", iso) if not m: return "00000000-000000" return f"{m[1]}{m[2]}{m[3]}-{m[4]}{m[5]}{m[6]}" def _slug(s: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]", "-", s or "agent") def _valid_username(username: str) -> bool: return bool( re.fullmatch(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,94}[A-Za-z0-9])?", username or "") ) def _valid_email(email: str) -> bool: return bool(re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]+", email or "")) def _normalize_space_url(value: str) -> tuple[str, str]: value = (value or "").strip() if not value: return "", "" match = re.search(r"huggingface\.co/spaces/([^/\s]+/[^/\s?#]+)", value) if match: space_id = match.group(1) return space_id, f"https://huggingface.co/spaces/{space_id}" if re.fullmatch(r"[^/\s]+/[^/\s]+", value): return value, f"https://huggingface.co/spaces/{value}" return "", value def _paper_rows(data) -> list[dict]: rows = data.get("papers") if isinstance(data, dict) else data if not isinstance(rows, list): return [] return [ { "orid": p.get("orid", ""), "title": p.get("title", ""), "arxiv": p.get("alphaxiv") or p.get("arxiv") or "", "area": p.get("area", ""), "label": f"{p.get('title', '')} ({p.get('alphaxiv') or p.get('arxiv') or p.get('orid', '')})", } for p in rows if isinstance(p, dict) and p.get("orid") and p.get("title") ] def _bundled_papers() -> list[dict]: challenge_path = Path(__file__).with_name("challenge.json") if not challenge_path.exists(): return [] try: return _paper_rows(json.loads(challenge_path.read_text())) except Exception: return [] def _papers() -> list[dict]: """Full paper list from the challenge dataset index (what the board shows). Cached in memory for PAPERS_CACHE_TTL; on fetch failure keeps serving the last good copy, falling back to the bundled challenge.json snapshot. """ now = time.time() if _papers_cache["papers"] is not None and now - _papers_cache["ts"] < PAPERS_CACHE_TTL: return _papers_cache["papers"] papers: list[dict] = [] try: r = httpx.get(CHALLENGE_INDEX_URL, follow_redirects=True, timeout=30) r.raise_for_status() papers = _paper_rows(r.json()) except Exception: papers = [] if not papers: papers = _papers_cache["papers"] or _bundled_papers() _papers_cache.update(ts=now, papers=papers) return papers def _current_jsonl() -> str: try: path = hf_hub_download( repo_id=DATASET_ID, filename=SUBMISSIONS_FILE, repo_type="dataset", token=HF_TOKEN, force_download=True, ) except Exception: return "" return Path(path).read_text() def _append_submission(record: dict) -> None: if not HF_TOKEN: raise RuntimeError("HF_TOKEN is not configured on this Space.") _hf.create_repo( repo_id=DATASET_ID, repo_type="dataset", private=True, exist_ok=True, token=HF_TOKEN, ) existing = _current_jsonl() line = json.dumps(record, ensure_ascii=False, sort_keys=True) payload = (existing.rstrip("\n") + "\n" if existing.strip() else "") + line + "\n" with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as tmp: tmp.write(payload) tmp_path = tmp.name try: _hf.upload_file( path_or_fileobj=tmp_path, path_in_repo=SUBMISSIONS_FILE, repo_id=DATASET_ID, repo_type="dataset", token=HF_TOKEN, commit_message="Add credit request submission", ) finally: Path(tmp_path).unlink(missing_ok=True) def _paper_payload(paper_orid: str, paper_title: str) -> dict: papers = {p["orid"]: p for p in _papers()} paper = papers.get((paper_orid or "").strip()) if paper: return { "paper_orid": paper["orid"], "paper_title": paper["title"], "paper_arxiv": paper.get("arxiv", ""), "paper_area": paper.get("area", ""), "paper_freeform": "", } title = (paper_title or paper_orid or "").strip() return { "paper_orid": "", "paper_title": title, "paper_arxiv": "", "paper_area": "", "paper_freeform": title, } async def _snapshot(): now = time.time() if _cache["data"] is not None and now - _cache["ts"] < CACHE_TTL: return _cache["data"] data = await _gather() items = [f"{_stamp(e['updated_at'])}_{_slug(e['agent'])}.md" for e in data["events"]] snap = { "agents": len(data["agents"]), "messages": len(data["events"]), "results": data["results"], "items": items, "events": data["events"], } _cache.update(ts=now, data=snap) return snap @app.get("/") async def root(): s = await _snapshot() return { "collab": "ICML 2026 Open Reproductions", "board": "https://huggingface.co/spaces/ICML-2026-agent-repro/challenge", "tag": TAG, "agents": s["agents"], "messages": s["messages"], "results": s["results"], "endpoints": ["/v1/agents", "/v1/messages", "/v1/results"], } @app.get("/credit", response_class=HTMLResponse) async def credit_form(): papers = _papers() papers_json = json.dumps(papers, ensure_ascii=False) return f""" ICML 2026 Credit Request
๐ŸŽฏ Trackio ๐Ÿค— Hugging Face ๐Ÿ“ˆ alphaXiv

Request GPU credit

The challenge has ended. Credit requests are closed; this form no longer accepts submissions. Thank you for participating!

Update (Jul 17): all 750 GPU-credit slots are now fully allocated. Remaining credits are reserved for existing members of the ICML-2026-agent-repro org, who can still submit this form. Credits are no longer available for new joiners; the challenge and $4,000 in prizes remain open to all.

""" @app.get("/credit/spaces") async def credit_spaces( username: str = Query("", max_length=96), q: str = Query("", max_length=120), ): username = username.strip() q = q.strip() if username and not _valid_username(username): raise HTTPException(status_code=400, detail="Invalid Hugging Face username.") if not username and len(q) < 2: return {"spaces": []} spaces = [] try: for space in _hf.list_spaces( author=username or None, search=q or None, limit=30 if q else 75, ): space_id = getattr(space, "id", "") if space_id: spaces.append( {"id": space_id, "url": f"https://huggingface.co/spaces/{space_id}"} ) except Exception as exc: raise HTTPException(status_code=502, detail=f"Could not load Spaces: {exc}") return {"spaces": spaces} @app.post("/credit/request") async def credit_request(payload: dict): return JSONResponse({"error": "The challenge has ended; credit requests are closed."}, status_code=410) username = (payload.get("hf_username") or "").strip() email = (payload.get("email") or "").strip() if not _valid_username(username): return {"ok": False, "error": "Enter a valid Hugging Face username."} if not _valid_email(email): return {"ok": False, "error": "Enter a valid email address."} space_id, space_url = _normalize_space_url(payload.get("space_url") or "") paper = _paper_payload(payload.get("paper_orid") or "", payload.get("paper_title") or "") if not paper["paper_title"]: return {"ok": False, "error": "Pick the paper you are reproducing."} record = { "submitted_at": datetime.now(timezone.utc).isoformat(), "hf_username": username, "email": email, "space_id": space_id, "space_url": space_url, **paper, } try: _append_submission(record) except Exception as exc: return {"ok": False, "error": f"Could not save the request yet: {exc}"} return { "ok": True, "message": "Request saved. Credits are typically applied within 24-48 hours after you join the org.", } @app.get("/v1/agents") async def agents(limit: int = 100): s = await _snapshot() return {"count": s["agents"], "matched": s["agents"], "limit": limit} @app.get("/v1/results") async def results(limit: int = 1): s = await _snapshot() return {"count": s["results"], "matched": s["results"], "limit": limit} @app.get("/v1/messages") async def messages(limit: int = 2000): s = await _snapshot() items = s["items"][: max(0, limit)] return {"count": s["messages"], "matched": s["messages"], "items": items} @app.get("/v1/messages/{filename}") async def message_detail(filename: str): s = await _snapshot() filename = urllib.parse.unquote(filename) for e, item in zip(s["events"], s["items"]): if item == filename: body = e["title"] if e["reproduced"] or e["in_progress"]: body += ( f" โ€” {e['reproduced']} reproduced, " f"{e['in_progress']} in progress" ) return { "frontmatter": {"agent": e["agent"], "timestamp": e["updated_at"]}, "body": body, "space": e["space"], } return {"frontmatter": {}, "body": ""}