diff --git a/CUSTOM_ENVS.md b/CUSTOM_ENVS.md index e21900797665e7b19cab84ae318c082c3a38c85b..7345aabca5862c746af59f289fec6d42bb37f6cc 100644 --- a/CUSTOM_ENVS.md +++ b/CUSTOM_ENVS.md @@ -107,6 +107,15 @@ can't load the renderer show. ## Running +Every summary and task also receives a versioned `support` descriptor from +`app/envs/capabilities.py`. It separates the source's framework, lifecycle, reward +semantics, transport and available capabilities. The UI and dataset MCP tools consume +the same descriptor. States are `available`, `external`, `unavailable`, and `unknown`; +only an implemented hosted runner can advertise execution here. Register a new +framework's documentation/lifecycle there and add its runner to `HOSTED_RUNNERS` only +after implementing dispatch and validation. See [EXPLORER_AUDIT.md](EXPLORER_AUDIT.md) +for the framework matrix and live protocol contract. + A run option is one way to run a task, by a **runner**: - `harbor`: OpenEnv's Harbor runner on an HF Sandbox (`app/runner.py`). Implement `materialize` (the task as a Harbor @@ -189,8 +198,10 @@ A processor can say more than a row: as dotted names, through lists. Put a format's set in `processors.HIDDEN` too: search never looks at them. **The framework readers.** NeMo Gym, Verifiers and verl / SkyRL rows are read by modules of their own, which show a -row as its framework sees it and say how to run it there (they never run here: the sandbox runner runs Harbor tasks, -and these need their framework's servers, sandboxes or trainers): +row as its framework sees it and say how to run it there. Verifiers, verl / SkyRL and most NeMo +configurations require their native servers, sandboxes or trainers. The two supported NeMo Pivot tool-simulation +configurations can also run here through Responses inference and NVIDIA's pinned argument verifier; they predict +function calls without executing the functions. `nemogym.run_option` validates this narrow hosted subset: | module | knows | how it's kept current | |---|---|---| diff --git a/app/admin.py b/app/admin.py index 3d3837c6eb70c49e21509ea3242587c3187ac8ab..68470bf5594db500bd0c8e0d6c060bf04d47bb7c 100644 --- a/app/admin.py +++ b/app/admin.py @@ -19,7 +19,7 @@ from typing import Any, Literal from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel, Field -from . import auth, catalog, config, runner, settings, store +from . import auth, auto_index, catalog, community, config, runner, search_api, settings, snapshot, space_checks, space_tasks, store STALE = 600 # a rollout whose record hasn't changed for this long is no longer live, whatever it says @@ -40,22 +40,29 @@ def _admin(request: Request) -> dict[str, Any]: @router.get("/overview") def overview(request: Request): _admin(request) - envs = catalog.environments(include_hidden=True) + envs = search_api.environments(include_hidden=True) runs = _runs() now = time.time() day = 86400 by_day = Counter(time.strftime("%Y-%m-%d", time.gmtime(r.get("created_at", 0))) for r in runs if now - r.get("created_at", 0) < 14 * day) finished = [r for r in runs if r.get("status") in ("done", "failed", "cancelled", "interrupted")] new = sorted((e for e in envs if e.get("created") and now - _ts(e["created"]) < 14 * day), key=lambda e: e["created"], reverse=True) + hidden_runs = set(settings.get("hidden_runs", [])) + ctx = search_api.context() + with snapshot.use() as (_, conn): + public_counts = search_api.sql_extra(conn, ctx, 0, search_api.Query(scope="ready"))["stats"] idx_files = list(config.INDEX_DIR.glob("*.json.gz")) pack_files = list(config.PACK_DIR.glob("*.json.gz")) if config.PACK_DIR.is_dir() else [] return { "environments": {"datasets": sum(1 for e in envs if e["kind"] == "dataset"), "spaces": sum(1 for e in envs if e["kind"] == "space"), - "openenv": sum(1 for e in envs if e.get("openenv")), "indexed": sum(1 for e in envs if e.get("indexed")), - "hidden": len(catalog.hidden()), "pinned": len(catalog.pinned()), "collections": len(catalog.collections())}, + "openenv": sum(space_checks.verified(e.get("api_check")) for e in envs), "indexed": sum(1 for e in envs if e.get("indexed")), + "checked_spaces": sum(space_checks.browseable(e.get("api_check")) for e in envs), + "public": public_counts, "hidden": len(catalog.hidden()), "pinned": len(catalog.pinned()), "collections": len(catalog.collections())}, "rollouts": {"total": len(runs), "live": sum(1 for r in runs if _live(r)), "today": sum(1 for r in runs if now - r.get("created_at", 0) < day), "week": sum(1 for r in runs if now - r.get("created_at", 0) < 7 * day), - "public": sum(1 for r in runs if r.get("visibility") == "public"), + "public": sum(community.is_public(r, hidden_runs) for r in runs), + "private": sum(r.get("visibility") != "public" for r in runs), + "by_runner": Counter(community.runner(r) for r in runs).most_common(), "failed": sum(1 for r in finished if r.get("status") != "done"), "finished": len(finished), "users": len({r.get("user") for r in runs}), "by_day": [[time.strftime("%Y-%m-%d", time.gmtime(now - i * day)), by_day.get(time.strftime("%Y-%m-%d", time.gmtime(now - i * day)), 0)] @@ -65,6 +72,8 @@ def overview(request: Request): "storage": {"indexes": len(idx_files), "index_bytes": sum(f.stat().st_size for f in idx_files), "packs": len(pack_files), "pack_bytes": sum(f.stat().st_size for f in pack_files)}, "recent": [_admin_view(r) for r in runs[:8]], + "live": [_admin_view(r) for r in runs if _live(r)][:24], + "operations": _operations(envs), "new": new[:12], "settings": _settings(), } @@ -117,6 +126,8 @@ def _admin_view(r: dict[str, Any]) -> dict[str, Any]: v = dict(r) if v.get("endpoint"): v["endpoint"] = {"host": v["endpoint"].get("host"), "model": v["endpoint"].get("model")} + v["runner"] = community.runner(r) + v["community"] = community.is_public(r, set(settings.get("hidden_runs", []))) v["live"] = _live(r) v["avatar"] = r.get("avatar") or avatar(r.get("user")) v["moderated"] = r["id"] in set(settings.get("hidden_runs", [])) @@ -125,14 +136,22 @@ def _admin_view(r: dict[str, Any]) -> dict[str, Any]: # ── rollouts ───────────────────────────────────────────────────────────────── @router.get("/rollouts") -def rollouts(request: Request, status: str = "", q: str = "", user: str = "", limit: int = 100, offset: int = 0): +def rollouts(request: Request, status: str = "", q: str = "", user: str = "", runner: str = "", + visibility: str = "", limit: int = 100, offset: int = 0): _admin(request) ql = q.strip().lower() - runs = [r for r in _runs() + hidden = set(settings.get("hidden_runs", [])) + all_runs = _runs() + runs = [r for r in all_runs if (not status or (status == "live" and _live(r)) or r.get("status") == status) and (not user or r.get("user") == user) - and (not ql or ql in f"{r.get('title', '')} {r.get('dataset', '')} {r.get('model', '')} {r.get('user', '')} {r['id']}".lower())] - return {"total": len(runs), "runs": [_admin_view(r) for r in runs[offset: offset + max(1, min(limit, 500))]]} + and (not runner or community.runner(r) == runner) + and (not visibility or (visibility == "community" and community.is_public(r, hidden)) + or (visibility == "moderated" and r["id"] in hidden) or r.get("visibility") == visibility) + and (not ql or ql in f"{r.get('title', '')} {r.get('dataset', '')} {r.get('model', '')} {r.get('user', '')} {r['id']} {community.runner(r)}".lower())] + offset = max(0, offset) + return {"total": len(runs), "runners": sorted({community.runner(r) for r in all_runs}), + "runs": [_admin_view(r) for r in runs[offset:offset + max(1, min(limit, 500))]]} @router.post("/rollouts/{run_id}/cancel") @@ -173,14 +192,14 @@ def people(request: Request): by: dict[str, dict[str, Any]] = {} for r in _runs(): name = r.get("user") or "unknown" - p = by.setdefault(name, {"name": name, "avatar": None, "runs": 0, "done": 0, "solved": 0, "live": 0, "public": 0, + p = by.setdefault(name, {"name": name, "avatar": None, "runs": 0, "done": 0, "scored": 0, "live": 0, "public": 0, "first": r.get("created_at"), "last": r.get("created_at"), "models": Counter(), "datasets": Counter()}) p["avatar"] = p["avatar"] or r.get("avatar") p["runs"] += 1 p["done"] += r.get("status") == "done" - p["solved"] += (r.get("reward") or 0) >= 0.999 + p["scored"] += r.get("status") == "done" and community.scalar_reward(r) p["live"] += _live(r) - p["public"] += r.get("visibility") == "public" + p["public"] += community.is_public(r, set(settings.get("hidden_runs", []))) p["first"] = min(p["first"] or 0, r.get("created_at") or 0) p["last"] = max(p["last"] or 0, r.get("created_at") or 0) p["models"][r.get("model") or "custom endpoint"] += 1 @@ -197,10 +216,10 @@ def people(request: Request): @router.get("/environments") def environments(request: Request): _admin(request) - envs = catalog.environments(include_hidden=True) + envs = search_api.environments(include_hidden=True) gone = catalog.hidden() runs = Counter(r.get("dataset") for r in _runs()) - return {"environments": [{**e, "hidden": e["key"] in gone, "rollouts": runs.get(e["id"], 0) if e["kind"] == "dataset" else 0} for e in envs], + return {"environments": [{**e, "hidden": e["key"] in gone, "ready": e["kind"] == "dataset" or space_checks.browseable(e.get("api_check")), "rollouts": runs.get(e["id"], 0) if e["kind"] == "dataset" else 0} for e in envs], "collections": catalog.collections(), "jobs": _jobs()} @@ -238,7 +257,7 @@ def env_action(body: EnvAction, request: Request): settings.save(u["name"], {"collections": cols}, note=f"{body.action} {key}" + (f" in {body.collection}" if body.collection else "")) elif body.action == "index": if key.startswith("space:"): - raise HTTPException(400, "Spaces aren't indexed") + raise HTTPException(400, "Space task counts are discovered automatically from their Task API; dataset indexes are separate") st = catalog.index_status(key) # anonymous: public datasets only settings.save(u["name"], {}, note=f"indexed {key}") return {"ok": True, "job": {k: v for k, v in st.items() if k != "index"}} @@ -339,4 +358,22 @@ def indexes(request: Request): out.append({"spec": f.name[:-8].replace("__", "/", 1), "bytes": f.stat().st_size, "pack_bytes": pack.stat().st_size if pack.exists() else None, "tasks": head.get("tasks"), "version": head.get("version"), "current": head.get("version") == catalog.INDEX_VERSION, "built": head.get("built")}) - return {"indexes": out, "version": catalog.INDEX_VERSION, "jobs": _jobs()} + return {"indexes": out, "version": catalog.INDEX_VERSION, "jobs": _jobs(), "operations": _operations()} + + +def _operations(envs=None): + """Evidence from the shared store, not the admin process's worker settings.""" + envs = envs if envs is not None else search_api.environments(include_hidden=True) + spaces = [e for e in envs if e["kind"] == "space"] + records = space_checks.inventory() + states = Counter(space_checks.status(records.get(e["id"])) for e in spaces) + versions = Counter(e.get("declared_version") or "Unknown" for e in spaces if space_checks.verified(records.get(e["id"]))) + job = auto_index.status() + now = time.time() + stale = bool(job and now > (job.get("next_at") or job.get("started_at") or 0) + auto_index.INTERVAL) + return {"snapshot": snapshot.status(), "indexer": {**job, "stale": stale}, + "checks": {"candidates": len(spaces), "ready": sum(space_checks.browseable(records.get(e["id"])) for e in spaces), + "openenv": sum(space_checks.verified(records.get(e["id"])) for e in spaces), + "statuses": dict(states), "versions": versions.most_common(), "fresh_seconds": space_checks.FRESH, + "latest_at": max((records.get(e["id"], {}).get("checked_at", 0) for e in spaces), default=0)}, + "task_catalog": space_tasks.census(records, [e["id"] for e in spaces if e["key"] not in catalog.hidden() and space_checks.browseable(records.get(e["id"]))])} diff --git a/app/admin_app.py b/app/admin_app.py index 6fcc02fc4c429f39313ae56cdff2a5bbb7e3b5fd..8ebb27f3f62877788078ffd858ed17a56b7d35e2 100644 --- a/app/admin_app.py +++ b/app/admin_app.py @@ -34,12 +34,13 @@ ADMIN_WEB = config.ROOT / "web-admin" def _startup() -> None: import threading - from . import catalog + from . import search_api, snapshot config.STORAGE_DIR.mkdir(parents=True, exist_ok=True) config.INDEX_DIR.mkdir(parents=True, exist_ok=True) # the Hub listing the dashboard opens with: fetched now, so the first visit doesn't wait on it - threading.Thread(target=lambda: catalog.environments(include_hidden=True), daemon=True, name="warm").start() + snapshot.start() + threading.Thread(target=lambda: search_api.environments(include_hidden=True), daemon=True, name="warm").start() @app.get("/healthz", include_in_schema=False) diff --git a/app/auto_index.py b/app/auto_index.py new file mode 100644 index 0000000000000000000000000000000000000000..9a274b802a2407298b5f93455fff6bb66c886c45 --- /dev/null +++ b/app/auto_index.py @@ -0,0 +1,131 @@ +"""Bounded automatic public catalog maintenance, independent of page visits. + +The existing indexer runs in a child process so stalled downloads cannot leave +builder threads behind. A process lease prevents duplicate app workers from +publishing concurrently. No HF Jobs, inference, or episode execution is used. +Deployments with an external indexer can set RLX_AUTO_INDEX=0. +""" +from __future__ import annotations + +import gzip +import json +import logging +import os +import subprocess +import sys +import threading +import time + +from . import catalog, config + +INTERVAL = 600 +LISTING_REFRESH = 3600 +MAX_SECONDS = 600 +_stop = threading.Event() +_worker = None +log = logging.getLogger("rlx") + + +def status(): + try: + record = json.loads((config.STORAGE_DIR / "auto-index.json").read_bytes()) + return record if isinstance(record, dict) else {} + except (OSError, ValueError): + return {} + + +def _save(record): + catalog.atomic_write(config.STORAGE_DIR / "auto-index.json", json.dumps(record).encode()) + + +def cached_listing(now): + """Reuse only a recent complete public discovery; never extend its age.""" + try: + with gzip.open(config.STORAGE_DIR / "listing.json.gz", "rt") as f: + doc = json.load(f) + if not isinstance(doc, dict): + return None, None + at = doc.get("at", 0) + if doc.get("full") and isinstance(doc.get("rows"), list) and doc["rows"] and 0 <= now - at < LISTING_REFRESH: + return doc["rows"], at + except (OSError, ValueError, TypeError): + pass + return None, None + + +def cycle(): + """One lease-protected cycle; publication remains atomic in indexer.run.""" + import fcntl + from . import indexer, snapshot + + config.CACHE_DIR.mkdir(parents=True, exist_ok=True) + with (config.CACHE_DIR / "auto-index.lock").open("a") as lease: + try: + fcntl.flock(lease, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return + now = time.time() + previous = status() + if previous.get("next_at", 0) > now: + return + record = {"state": "indexing", "started_at": now, "last_success": previous.get("last_success"), + "failures": previous.get("failures", 0), "next_at": now + MAX_SECONDS + INTERVAL} + _save(record) + try: + rows, at = cached_listing(now) + report = indexer.run(snapshot.open_store(), rows=rows, listing_at=at, budget_s=90, max_builds=4) + _save({"state": "idle", "last_success": time.time(), "next_at": time.time() + INTERVAL, + "failures": 0, "counts": report["counts"]}) + except Exception as exc: + failures = record["failures"] + 1 + _save({**record, "state": "retrying", "failures": failures, "error": type(exc).__name__, + "next_at": time.time() + min(3600, INTERVAL * 2 ** min(failures - 1, 3))}) + log.warning("Automatic catalog update failed: %s", type(exc).__name__) + + +def _child(): + # A public index must not accidentally depend on a developer's HF login. + env = {**os.environ, "HF_HUB_DISABLE_IMPLICIT_TOKEN": "1"} + child = subprocess.Popen([sys.executable, "-m", "app.auto_index"], env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + deadline = time.monotonic() + MAX_SECONDS + try: + while child.poll() is None: + if _stop.wait(1) or time.monotonic() >= deadline: + break + finally: + if child.poll() is None: + child.terminate() + try: + child.wait(timeout=3) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + + +def start(): + global _worker + if os.environ.get("RLX_TEST_TMP") or os.environ.get("RLX_AUTO_INDEX", "1") != "1" or (_worker and _worker.is_alive()): + return + _stop.clear() + def watch(): + while not _stop.is_set(): + try: + _child() + except Exception as exc: + log.warning("Automatic catalog worker failed: %s", type(exc).__name__) + _stop.wait(60) + _worker = threading.Thread(target=watch, daemon=True, name="auto-index") + _worker.start() + + +def stop(): + _stop.set() + if _worker: + _worker.join(timeout=5) + + +if __name__ == "__main__": + from . import runtime + runtime.hub_timeouts() + cycle() diff --git a/app/catalog.py b/app/catalog.py index dd9973037bd907ad9d45f8c3adf1251073c58763..82e5ac71789d408adbfb058a1c686717e1a24b9d 100644 --- a/app/catalog.py +++ b/app/catalog.py @@ -92,7 +92,7 @@ def hidden() -> set[str]: return set(settings.get("hidden", [])) -# FineEnvs' flagship environment servers lead Trending until an admin saves pins of their own (an empty list included) +# Editorial picks, shown separately from the metric-based trending order. DEFAULT_PINNED = ["space:FineEnvs/geoguesser-env", "space:FineEnvs/latex-ocr-env"] @@ -263,7 +263,9 @@ def _listing_fetch(full: bool) -> list[dict[str, Any]]: for d in _api().list_datasets(filter=tag, sort="trending_score", limit=None if full else 1000, expand=_EXPAND): if d.id not in rows and not d.private and not d.gated: rows[d.id] = _summary(d) - except Exception: # noqa: BLE001 - one tag failing leaves the rest + except Exception: # noqa: BLE001 - only the quick bootstrap tolerates incomplete discovery + if full: + raise continue missing = [s for s in featured_datasets() if s not in rows] with ThreadPoolExecutor(max_workers=12) as pool: # one Hub call each: in parallel @@ -361,9 +363,6 @@ def environments(include_hidden: bool = False) -> list[dict[str, Any]]: indexed = headline(r["id"]) if r["kind"] == "dataset" else None fw = "harbor" if indexed and indexed.get("tasks") else r.get("framework") # its index found task folders coll = collection_of(r["key"]) - if r["kind"] == "space" and coll == "fineenvs" and fw != "openenv": # curated as OpenEnv servers, manifest or not - r = {**r, "openenv": True, "badges": ["OpenEnv"]} - fw = "openenv" out.append({**r, "framework": fw, "collection": coll, "pinned": r["key"] in pins, "indexed": indexed}) return out @@ -374,12 +373,12 @@ _SPACE_EXPAND = ["likes", "trendingScore", "sdk", "runtime", "tags", "lastModifi def _space_summary(s: Any) -> dict[str, Any]: card = (s.card_data.to_dict() if s.card_data else {}) or {} - raw = s.tags or [] + raw = [str(t).lower() for t in (s.tags or [])] stage = getattr(s.runtime, "stage", None) if s.runtime is not None else None version = next((t[8:] for t in raw if re.match(r"^openenv-\d", t)), None) files = [f.rfilename for f in (getattr(s, "siblings", None) or [])] manifest = min((f for f in files if _MANIFEST.match(f) and not _VENDORED.match(f)), key=lambda f: f.count("/"), default=None) - openenv = "openenv" in raw or bool(version) or manifest is not None + openenv = bool({"openenv", "library:openenv"} & set(raw)) or bool(version) or manifest is not None ors = bool({"ors", "openreward"} & set(raw)) return {"id": s.id, "key": f"space:{s.id}", "kind": "space", "openenv": openenv, "manifest": manifest, "framework": "openenv" if openenv else "ors" if ors else "space", "files": len(files) or None, @@ -395,10 +394,10 @@ def _space_summary(s: Any) -> dict[str, Any]: "private": bool(s.private), "gated": False} -SPACE_TAGS = ("openenv", "rl-environment", "ors", "openreward") +SPACE_TAGS = ("openenv", "library:openenv", "rl-environment", "ors", "openreward") # a manifest a Space's own environment declares (not one vendored from OpenEnv's source, or its templates) _MANIFEST = re.compile(r"^(?:[^/]+/)?openenv\.ya?ml$") -_VENDORED = re.compile(r"^(src/(openenv|core)/|build/|.*\.egg-info/|envs/[^/]+/)") +_VENDORED = re.compile(r"^(src/|build/|dist/|templates?/|examples?/|tests?/|vendor/|\.venv/|.*\.egg-info/|envs/)") def spaces(full: bool = True) -> list[dict[str, Any]]: @@ -415,12 +414,16 @@ def spaces(full: bool = True) -> list[dict[str, Any]]: if sp.private or sp.sdk != "docker" or "research-article-template" in tags: continue rows.setdefault(sp.id, _space_summary(sp)) - except Exception: # noqa: BLE001 - Spaces failing leaves the datasets + except Exception: # noqa: BLE001 - a full census must not silently publish partial counts + if full: + raise continue for sid in featured: if sid not in rows: try: - rows[sid] = _space_summary(_api().space_info(sid, expand=expand)) # with its files: its manifest + sp = _api().space_info(sid, expand=expand) + if not sp.private and sp.sdk == "docker" and "research-article-template" not in (sp.tags or []): + rows[sid] = _space_summary(sp) except Exception: # noqa: BLE001 continue return list(rows.values()) diff --git a/app/community.py b/app/community.py new file mode 100644 index 0000000000000000000000000000000000000000..1c87e8f3c1d5d8f35c4eb6d0b19c6e879bbd0503 --- /dev/null +++ b/app/community.py @@ -0,0 +1,98 @@ +"""Runner-independent filtering and honest summaries of saved public rollouts. + +Visibility is checked by the API before records enter this module. Facets ignore +their own selection; headline counts always describe the filtered population. +Rewards are grouped by environment, runner, domain, metric, judge and revision. +There is no cross-environment model ranking. +""" +from __future__ import annotations + +import math +from collections import Counter, defaultdict + +DOMAINS = {"code": "Code", "webdev": "Webdev", "cyber": "Cyber", "music": "Music", "general": "General"} + + +def is_public(r, hidden=()): + return (r.get("visibility") == "public" and r.get("status") == "done" and scalar_reward(r) + and not r.get("restricted") and r.get("id") not in hidden) + + +def scalar_reward(r): + value = r.get("reward") + try: + return type(value) in (int, float) and math.isfinite(value) + except OverflowError: + return False + + +def runner(r): + value = r.get("runner") or ("mimo" if r.get("domain") in DOMAINS else "harbor") + return {"nemo": "nemo-gym", "nemogym": "nemo-gym", "nemo_gym": "nemo-gym"}.get(value, value) + + +def group(r): + return DOMAINS.get(r.get("domain")) or r.get("dataset") or "Unknown environment" + + +def served(r): + return "own endpoint" if r.get("endpoint") else r.get("provider") or "auto" + + +def band(r): + if r.get("status") != "done" or not scalar_reward(r): + return "unscored" + return "full" if r["reward"] >= .999 else "zero" if r["reward"] <= 0 else "partial" + + +VALUES = {"dataset": lambda r: r.get("dataset") or "", "group": group, "runner": runner, + "model": lambda r: r.get("model") or "custom endpoint", "served": served, + "judge": lambda r: r.get("judge") or "", "reward": band, + "thinking": lambda r: (r.get("params") or {}).get("thinking") or "default"} + + +def context(r): + return (r.get("dataset") or "Unknown environment", runner(r), r.get("domain") or "", + r.get("reward_key") or "reward", r.get("judge") or "", + r.get("sha") or (r.get("provenance") or {}).get("commit") or "") + + +def stats(runs): + groups = defaultdict(list) + for r in runs: + if r.get("status") == "done" and scalar_reward(r): + groups[(*context(r), VALUES["model"](r), served(r))].append(r["reward"]) + results = [{"dataset": k[0], "runner": k[1], "domain": k[2], "metric": k[3], "judge": k[4], "revision": k[5], + "model": k[6], "served": k[7], "runs": len(v), "mean": math.fsum(x / len(v) for x in v), "min": min(v), "max": max(v)} + for k, v in sorted(groups.items())] + contexts = {context(r) for r in runs} + return {"runs": len(runs), "scored": sum(len(v) for v in groups.values()), "datasets": len({r.get("dataset") for r in runs}), + "tasks": len({(r.get("dataset"), r.get("path")) for r in runs}), "models": len({VALUES["model"](r) for r in runs}), + "runners": len({runner(r) for r in runs}), "comparable": len(contexts) == 1, "results": results} + + +def query(everyone, filters, *, sort="new", offset=0, limit=100): + def keep(r, skip=None): + if filters.get("path") is not None and r.get("path") != filters["path"]: + return False + words = str(filters.get("q") or "").lower().split()[:16] + haystack = " ".join(str(r.get(k) or "") for k in ("title", "task_id", "path", "dataset", "model", "runner", "adapter")).lower() + if not all(word in haystack for word in words): + return False + return all(not value or name == skip or VALUES[name](r) == value + for name, value in filters.items() if name in VALUES) + runs = [r for r in everyone if keep(r)] + # Stable explicit ordering also makes pagination deterministic for tied times. + runs.sort(key=lambda r: (-(r.get("created_at") or 0), r["id"])) + key = {"reward_desc": lambda r: -r["reward"], "reward_asc": lambda r: r["reward"], + "cost_asc": lambda r: (r.get("cost") or {}).get("total") or 0, + "cost_desc": lambda r: -((r.get("cost") or {}).get("total") or 0)}.get(sort) + if key: + runs.sort(key=key) + facets = {} + for name, value in VALUES.items(): + counts = Counter(value(r) for r in everyone if keep(r, name) and value(r)) + facets[name] = [[k, n] for k, n in sorted(counts.items(), key=lambda x: (-x[1], x[0]))] + summary = stats(runs) + return {"stats": summary, "facets": facets, "total": len(runs), "tasks": summary["tasks"], + "runs": runs[max(0, offset):max(0, offset) + max(1, min(limit, 500))]} diff --git a/app/config.py b/app/config.py index 434c521b828ec88d4165eea8f548d2dee0e13725..fc6a241b6eb204b492b8598ddf65735db1a8e419 100644 --- a/app/config.py +++ b/app/config.py @@ -23,6 +23,8 @@ OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface SPACE_HOST = os.environ.get("SPACE_HOST", "") PUBLIC_URL = (os.environ.get("RLX_PUBLIC_URL") or (f"https://{SPACE_HOST}" if SPACE_HOST else "")).rstrip("/") LOCAL_MODE = not OAUTH_CLIENT_ID +# Enable only behind a proxy that replaces/appends the visitor's address. +TRUST_PROXY = os.environ.get("RLX_TRUST_PROXY", "1" if os.environ.get("SPACE_ID") else "0") == "1" # `jobs` runs the sandbox on the visitor's account, `inference-api` calls Inference Providers as them, # `read-repos` opens their private and gated datasets. OAUTH_SCOPES = os.environ.get("RLX_OAUTH_SCOPES", "openid profile inference-api jobs read-repos").split() # the admin Space asks for less diff --git a/app/envs/capabilities.py b/app/envs/capabilities.py new file mode 100644 index 0000000000000000000000000000000000000000..cbe4d532de9a6436ff2d600c750e23dee513e6f6 --- /dev/null +++ b/app/envs/capabilities.py @@ -0,0 +1,127 @@ +"""Evidence-based capabilities shared by dataset adapters, live Spaces, UI and MCP. + +A framework label describes a format, not permission or proof that it can run. +A dataset's native harness may be external; a live server may be asleep or may +speak HTTP instead of OpenEnv's WebSocket protocol. Keep those axes independent. +The contract is additive so older clients can keep using run.options/step_api. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + + +@dataclass(frozen=True) +class Framework: + id: str + label: str + task: str + lifecycle: str + reward: str + docs: str + + +FRAMEWORKS = { + f.id: f for f in ( + Framework("harbor", "Harbor", "Task folder or packed task files", + "Prepare sandbox → run agent → verify → collect artifacts", + "The task's verifier writes a scalar or named reward metrics.", + "https://www.harborframework.com/docs/tasks"), + Framework("openenv", "OpenEnv", "Live environment with typed actions and observations", + "Open session → reset → step until done → close", + "The environment returns rewards and termination with observations.", + "https://huggingface.co/docs/openenv/main/reference/core"), + Framework("verifiers", "Verifiers", "Rows loaded by an installed environment or taskset", + "Load environment/taskset → run its harness → score the rollout", + "The installed environment's rubric or judge defines the reward; rows alone do not include that runtime.", + "https://github.com/PrimeIntellect-ai/verifiers/tree/main/docs"), + Framework("nemo-gym", "NeMo Gym", "Task data plus agent and resources-server configuration", + "Seed → interact → verify, or reset → step for Gymnasium servers", + "A resources server or embedded agent harness computes the reward.", + "https://docs.nvidia.com/nemo/gym/build-verifiers/"), + Framework("verl", "verl", "Training rows with prompt and reward metadata", + "Generate responses → route to the configured reward function → train", + "data_source selects a scorer unless training config supplies a custom reward function.", + "https://verl.readthedocs.io/en/latest/advance/reward_loop.html"), + Framework("skyrl", "SkyRL", "Training rows routed to a skyrl-gym environment", + "Load rows → run environment interactions → compute training rewards", + "The selected environment and its configuration define the reward.", + "https://skyrl.readthedocs.io/en/latest/"), + Framework("mimo", "MiMo", "Tasks from the MiMo RL environment release", + "Prepare task → run the release's domain harness → grade", + "Each domain uses its own verifier; supported conversions also run through Harbor.", + "https://huggingface.co/datasets/XiaomiMiMo/MiMo-V2.6-RL-oss"), + Framework("gymnasium", "Gymnasium HTTP", "A server exposing reset and step over HTTP", + "Reset → step until terminated or truncated → close", + "The server returns step rewards; its HTTP schema defines the request and response.", + "https://gymnasium.farama.org/api/env/"), + Framework("mcp", "MCP", "Tools exposed by a live server", "Connect → call advertised tools → close", + "A reward is available only when the server exposes a grader or returns one.", + "https://modelcontextprotocol.io/specification/2025-06-18/server/tools"), + Framework("ors", "ORS", "A server with named environments and sessions", + "Create session → call environment → close session", "The server defines its grading contract.", ""), + Framework("api", "HTTP API", "A live server's published operations", "Call the server's documented routes", + "Inspect the published schema and grading routes for this server.", ""), + Framework("rows", "RL dataset", "Dataset rows", "Browse task data; use the source's environment to execute it", + "A dataset label or answer column alone does not establish a runnable grader.", ""), + ) +} +HOSTED_RUNNERS = frozenset({"harbor", "mimo", "nemo-gym"}) + + +def framework(value: str | None) -> Framework: + key = str(value or "").lower().replace("_", "-").replace(" ", "-") + aliases = {"nemo": "nemo-gym", "nemogym": "nemo-gym", "generic": "rows", "rl": "rows", "harbor-(packed)": "harbor"} + key = aliases.get(key, key) + if key.startswith("mimo"): + key = "mimo" + return FRAMEWORKS.get(key, FRAMEWORKS["rows"]) + + +def capability(id: str, label: str, state: str, detail: str) -> dict[str, str]: + if state not in {"available", "external", "unavailable", "unknown"}: + raise ValueError(f"invalid capability state: {state}") + return {"id": id, "label": label, "state": state, "detail": detail} + + +def dataset_support(name: str, *, state: str = "ready", options: list[dict[str, Any]] | None = None, + note: str = "", revision: str | None = None, restricted: bool = False) -> dict[str, Any]: + f = framework(name) + ready = state == "ready" + features = [capability("browse", "Browse tasks", "available" if ready else "unknown", + "Inspect the prompt, grading information and source files." if ready else "Task discovery is still in progress.")] + if options is None: + features.append(capability("run", "Run an agent", "unknown", "Open a task to check its harness and runtime requirements.")) + else: + runnable = [o for o in options if o.get("ok") and o.get("runner") in HOSTED_RUNNERS and not o.get("href")] + external = [o for o in options if o.get("href") or (not o.get("ok") and o.get("runner") in {"verifiers", "nemo-gym", "nemogym", "verl", "skyrl"})] + why = "; ".join(dict.fromkeys(o.get("why") for o in options if o.get("why"))) + features.append(capability("run", "Run an agent", "available" if runnable else "external" if external else "unavailable", + "Runs here with " + ", ".join(o["label"] for o in runnable) + "." if runnable else + why or note or "This source has no executable harness configured here.")) + features.append(capability("mcp", "Explore with an agent", "unavailable" if restricted else "available" if ready else "unknown", + "The MCP bridge only serves public datasets; this source requires your sign-in." if restricted else + "MCP exposes public task browsing. Execution follows the runtime support above.")) + return {"version": 1, "framework": asdict(f), "source": "dataset", "evidence": "adapter", "revision": revision, + "transport": None, "capabilities": features} + + +def space_support(info: dict[str, Any]) -> dict[str, Any]: + up = bool(info.get("running")) + f = framework(info.get("framework") or "api") + transport = "openenv-ws" if info.get("openenv") else "http" if info.get("api") else "mcp-http" if info.get("mcp") else None + can_play = bool(info.get("step_api") or info.get("mcp") or info.get("api")) + status = "available" if up and can_play else "unavailable" if up else "unknown" + detail = ("Play an episode over OpenEnv's WebSocket session." if transport == "openenv-ws" else + "Call the published HTTP routes; each explorer session keeps its own cookies." if transport == "http" else + "Call MCP tools. Episode state depends on the server's MCP implementation.") if can_play else "No supported interactive API was discovered." + if not up: + detail = "The Space is " + str(info.get("stage") or "unreachable").lower().replace("_", " ") + "; check its live interface when it is running." + return {"version": 1, "framework": asdict(f), "source": "space", "evidence": "live" if up else "unverified", + "transport": transport, "capabilities": [ + capability("play", "Interactive playground", status, detail), + capability("browse", "Browse tasks", "available" if up and info.get("task_api") else "unavailable" if up else "unknown", + "Tasks are served by the environment's Task API." if info.get("task_api") else "No live Task API was discovered; source files may describe the tasks."), + capability("mcp", "Connect an agent", status, + "The explorer bridges the advertised interface to MCP." if up and can_play else detail), + ]} diff --git a/app/envs/mimo.py b/app/envs/mimo.py index e980b17fcbbf503752fa0f9c8cbdc028730286ac..d6bf536c2e850cacd702cd8858aac92fc78e0b62 100644 --- a/app/envs/mimo.py +++ b/app/envs/mimo.py @@ -250,7 +250,7 @@ class MiMoAdapter(c.Adapter): if not rows and not how: return None net = ("Web search and fetch tools are off." if d == "webdev" else "Web search and fetch tools are off, and the sites where answers live " - "(code hosting, bug trackers, search engines) are unreachable from the sandbox; everything else, including local services, works as usual.") + "(code hosting, bug trackers, search engines) are added to a hosts-file blocklist. This reduces answer leakage but is not an egress firewall. Other connections depend on the hosting provider's network policy.") return c.section("setup", "How the sandbox is set up", [c.kv(rows), c.note(f"{how} {net}", "shield") if how else None], icon="box") def _glance(self, v: dict[str, Any]) -> list[list[str]]: diff --git a/app/envs/nemo_gym_build.py b/app/envs/nemo_gym_build.py index 9a5c17ac0910e4af1d9c69d60266f502a3e0024b..f50f905440a6afe5f5c33de2b14ba674f361e695 100644 --- a/app/envs/nemo_gym_build.py +++ b/app/envs/nemo_gym_build.py @@ -18,6 +18,12 @@ from pathlib import Path import yaml +# Reviewed against each server's implementation; the upstream environment table +# incorrectly calls these LLM judges. Keep regenerated catalogs accurate too. +VERIFICATION_OVERRIDES = { + "instruction_following": "Rule-based instruction checks", + "math_proof_judgement": "Deterministic answer parsing", +} def main(ROOT: Path) -> None: OUT = CAT = Path(__file__).with_name("nemo_gym_catalog.json") @@ -93,6 +99,8 @@ def main(ROOT: Path) -> None: continue t = table.get(d.name, {}) servers[d.name] = {k: v for k, v in {"description": t.get("description"), "verification": t.get("verification"), "readme": readme_para(d)}.items() if v} + if d.name in VERIFICATION_OVERRIDES: + servers[d.name]["verification"] = VERIFICATION_OVERRIDES[d.name] for cfg in sorted((d / "configs").glob("*.yaml")): flavor = d.name if cfg.stem == d.name else f"{d.name}/{cfg.stem}" scan(cfg, d.name, flavor, None) diff --git a/app/envs/nemo_gym_catalog.json b/app/envs/nemo_gym_catalog.json index 2f8a04ec34ad558e0824fab67537146c80b75c0d..2e525e6438caffd4d4937ac047f9334729b033fc 100644 --- a/app/envs/nemo_gym_catalog.json +++ b/app/envs/nemo_gym_catalog.json @@ -1985,7 +1985,7 @@ }, "instruction_following": { "description": "IFEval and IFBench-style instruction following", -"verification": "LLM judge", +"verification": "Rule-based instruction checks", "readme": "Evaluates language model responses against instruction-following criteria using Open-Instruct and IFEval constraints.", "fields": { "id": { @@ -2454,7 +2454,7 @@ }, "math_proof_judgement": { "description": "Binary proof judgement — model reads a problem and proof, outputs Yes/No", -"verification": "LLM judge", +"verification": "Deterministic answer parsing", "readme": "Binary judgement of math proofs. The policy model reads a problem + candidate proof and outputs `Judgement: Yes` (proof correct) or `Judgement: No` (proof incorrect). Verification is fully deterministic — a single regex parses the final judgement — so **no external LLM judge is called**. This mirrors NeMo Skills' `answer-judgement` metric (`is_correct_judgement` format 1).", "fields": { "expected_judgement": { diff --git a/app/envs/nemogym.py b/app/envs/nemogym.py index 5fef20e7ec4c1456f8b98aed8faff7670b4ae536..a0a8f16a1dff4f80887df2b1492575a8dbeaa98d 100644 --- a/app/envs/nemogym.py +++ b/app/envs/nemogym.py @@ -250,7 +250,8 @@ def run_blocks(spec: str, config: str, split: str, agents: list[dict[str, Any]], artifact = next((u.get("artifact") for u in cat["datasets"].get(spec.lower()) or [] if u.get("type") == split and u.get("artifact")), None) \ if config == "default" else None what = f"this dataset's {split} split" + ("" if config == "default" else f" of {config}") - lines = ["# NeMo Gym, from source (Python 3.13+, uv)", "git clone https://github.com/NVIDIA-NeMo/Gym.git && cd Gym", + lines = ["# NeMo Gym, from the revision used by this explorer's catalog (Python 3.13+, uv)", + "git clone https://github.com/NVIDIA-NeMo/Gym.git && cd Gym", f"git checkout {cat['commit']}", "uv venv --python 3.13.14 && source .venv/bin/activate && uv sync", "", f"# the rows: {what}, as JSON Lines" + (", and this row alone" if row is not None else ""), *_download(spec, config, split, local, artifact, rowfiles.files_of(spec, meta or {}, config, split) if config != "default" else [])] @@ -292,8 +293,21 @@ def run_blocks(spec: str, config: str, split: str, agents: list[dict[str, Any]], return blocks -def run_option(agent: dict[str, Any]) -> dict[str, Any]: - """Not here: the run panel says why in one sentence and points at the commands on the page.""" +def run_option(agent: dict[str, Any], row: dict | None = None) -> dict[str, Any]: + from .. import nemo_runner + if row is not None and nemo_runner.supported(agent): + try: + nemo_runner.validate(row, agent) + except (ValueError, KeyError, TypeError) as error: + return c.run_option("nemo-gym", "NeMo Gym prediction", ok=False, why=str(error), endpoint=False, sandbox=False) + return c.run_option("nemo-gym", "NeMo Gym prediction", ok=True, endpoint=False, sandbox=False, + about="One model prediction, scored by NVIDIA's NeMo Gym argument comparator. Tool calls are predictions, not executed actions. No sandbox is needed.", + notes=[f"Verifier pinned to NVIDIA-NeMo/Gym `{nemo_runner.REVISION[:12]}` with its native 0.1 word-similarity threshold."] + + (["Some source tools have invalid strict schemas. Their parameters stay unchanged; strict enforcement is disabled so HF providers accept them."] if nemo_runner.relaxed_tools(row) else []), + harnesses=["opencode"], fields=[ + c.field("max_tokens", "Max output tokens", "number", default=2048, min=256, max=8192, advanced=True), + c.field("temperature", "Temperature", "number", min=0, max=2, step=0.1, advanced=True)], + estimate={"tokens_in": 4000, "tokens_out": 1000, "minutes": 0}) rs = agent.get("resources_server") return c.run_option("nemo-gym", "NeMo Gym", ok=False, why=f"it is scored by NeMo Gym's {rs + ' ' if rs else ''}resources server, which runs only inside NeMo Gym", @@ -430,8 +444,9 @@ class NemoGym(Processor): rows = convo.field_rows({**dict(settings), **nested}) sections.append(section("request", "Request settings", [c.custom("rl", "fields", {"rows": rows}, convo.fields_text(rows))], kind="blocks", note="the rest of responses_create_params")) + option = run_option(agent, row) sections.append(section("run", "Run it with NeMo Gym", run_blocks(ds.spec, config, split, [agent], row=i, meta=ds.card), kind="blocks", - note="it doesn't run in this app")) + note="Native CLI alternative; prediction runs are also available here" if option["ok"] else "This task needs the native runtime")) rest = {k: v for k, v in clean.items() if k not in task_data and k not in FRAMEWORK and v not in (None, "", [], {})} if rest: label = "provenance and metrics" if roles_of else "the rest of the row" @@ -452,7 +467,7 @@ class NemoGym(Processor): glance.append(["Pass rate", p]) card = self.card(row, i, roles) return {"title": title[:200], "id": card["id"], "chips": card["chips"], "sections": sections, "withheld": gone, "glance": glance, - "framework_run": run_option(agent), + "framework_run": option, "summary": "Scored on the next action matching the expert's" if pivot else (f"Verifier: {agent['verification'].lower()}" if agent.get("verification") else "Scored by its resources server")} diff --git a/app/envs/registry.py b/app/envs/registry.py index 5e31c6375b89f475c5d10ccae771d706b920cf98..89b2e1753d89174c775efc4939529f378792cb5a 100644 --- a/app/envs/registry.py +++ b/app/envs/registry.py @@ -16,6 +16,7 @@ from typing import Any, Callable from .. import catalog from .contract import Adapter, Env +from .capabilities import dataset_support from .harbor import HarborAdapter from .mimo import MiMoAdapter, harbor_aliases, harbor_to_mimo from .rows import RowsAdapter @@ -86,8 +87,10 @@ def summary(spec: str, token: str | None = None, subset: str | None = None) -> d env = resolve("dataset", spec, token) s = env.adapter.summary(env, subset) how = s.get("how") or {} + name = how.get("framework") or env.adapter.framework return {"env": {**describe_env(env), "framework": how.get("framework") or env.adapter.framework}, - "info": {k: v for k, v in env.meta.items() if k != "all_tags"}, "collection": catalog.collection(spec), **s} + "info": {k: v for k, v in env.meta.items() if k != "all_tags"}, "collection": catalog.collection(spec), **s, + "support": dataset_support(name, state=s.get("state", "ready"), revision=env.meta.get("sha"), restricted=bool(env.meta.get("restricted")))} def tasks(spec: str, token: str | None = None, **kw: Any) -> dict[str, Any]: @@ -108,6 +111,7 @@ def task(spec: str, ref: str, token: str | None = None) -> dict[str, Any]: out.setdefault("collection", catalog.collection(spec)) out["env"] = {**describe_env(env), "framework": view.get("framework") or env.adapter.framework} out["run"] = {"options": options, "note": "" if options else env.adapter.run_note} + out["support"] = dataset_support(out["env"]["framework"], options=options, note=env.adapter.run_note, revision=env.meta.get("sha"), restricted=bool(env.meta.get("restricted"))) out["aliases"] = [{"env": k, "ref": r} for k, r in aliases(env, ref)] return out diff --git a/app/envs/rows.py b/app/envs/rows.py index f7087aaa74fbf19697a40f5694249e97fd6c6ca6..b155496f45ba00a307b2bef46eabb7ade60879f7 100644 --- a/app/envs/rows.py +++ b/app/envs/rows.py @@ -635,6 +635,16 @@ class RowsAdapter(c.Adapter): config, split, i = parse_ref(ref) view = self.task(env, ref) + if (view.get("_framework_run") or {}).get("runner") == "nemo-gym": + from .nemogym import agent_of + from .. import nemo_runner + *_, row, truncated = _row(env.id, config, split, i, env.token) + if truncated: + raise ValueError("This task row is truncated; use the complete dataset with NeMo Gym") + agent = agent_of(row, env.id) + nemo_runner.validate(row, agent) + return {"title": view["title"], "restricted": bool(env.meta.get("restricted")), "sha": env.meta.get("sha"), + "native_task": {"row": row, "agent": agent}} if view.get("_packed_run"): p = packed(env.id, config, split, i, env.token) return {"title": p["title"], "restricted": p["restricted"], "sha": p["sha"], "bytes": p["bytes"], "runnable": p["runnable"], diff --git a/app/envs/tools.py b/app/envs/tools.py index d46479dc1c4dec9bac27b3585aeb4960fae23023..a4381a338b1e2cd3abcb81886d480e19418a07eb 100644 --- a/app/envs/tools.py +++ b/app/envs/tools.py @@ -49,6 +49,7 @@ def call(spec: str, name: str, args: dict[str, Any]) -> Any: if s.get("state") != "ready": return {"state": s.get("state"), "note": "Its index is being built; try again in a minute.", "progress": s.get("progress")} return {"id": spec, "format": s["env"]["framework"], "read_as": (s.get("how") or {}).get("name"), "about": (s.get("how") or {}).get("about"), + "support": s.get("support"), "tasks": s.get("total"), "subsets": [x["id"] for x in s.get("subsets") or []], "subset": s.get("subset"), "facets": {f["key"]: f["label"] for f in s.get("facets") or []}, "overview": blocks_text([b for sec in s.get("overview") or [] for b in [{"type": "markdown", "text": f"## {sec['title']}"}, *sec["blocks"]]])} @@ -101,6 +102,10 @@ def task_text(v: dict[str, Any]) -> str: if v.get("withheld"): out.append("\nWithheld, as they hold the answer: " + ", ".join(v["withheld"])) opts = (v.get("run") or {}).get("options") or [] + support = v.get("support") or {} + if support.get("framework"): + f = support["framework"] + out.append(f"\nLifecycle: {f['lifecycle']}\nReward: {f['reward']}") out.append("\n## Running it\n" + ("\n".join(f"- {o['label']}: " + ("runs here" if o["ok"] else f"can't run here ({o['why']})") for o in opts) or (v.get("run") or {}).get("note") or "")) for link in v.get("links") or []: diff --git a/app/envs/verifiers_rows.py b/app/envs/verifiers_rows.py index 6bb42508cb8f260585c445f478f852c8dd3eb413..c18755b38fe2bfd32c2fd5f741a5b566c103701a 100644 --- a/app/envs/verifiers_rows.py +++ b/app/envs/verifiers_rows.py @@ -196,8 +196,11 @@ def _prompt_turns(row: dict[str, Any], env: dict[str, Any]) -> list[dict[str, An ts = [{"kind": "message", "role": "user", "text": row["question"]}] else: ts = [] - if env.get("system") and not any(t["role"] == "system" for t in ts): - ts.insert(0, {"kind": "message", "role": "system", "text": env["system"]}) + # v1 TaskData carries an explicit system_prompt. It belongs in the agent's + # conversation, ahead of the row's prompt, and overrides a known v0 default. + system = row.get("system_prompt") if isinstance(row.get("system_prompt"), str) else env.get("system") + if system and not any(t["role"] == "system" for t in ts): + ts.insert(0, {"kind": "message", "role": "system", "text": system}) return ts @@ -308,7 +311,7 @@ class Verifiers(Processor): gblocks += [c.note("`info`, which the rewards also get:", "list"), c.custom("rl", "fields", {"rows": rows}, convo.fields_text(rows))] sections.append(section("grading", "How it's graded", gblocks, kind="blocks", note="the environment's rewards")) sections.append(section("run", "Run it with Verifiers", run_blocks(ds.spec, env, config, split, row=i), kind="blocks", note="it doesn't run in this app")) - shown = {"problem_statement", "prompt", "question", "info", "task", *IMAGE_COLS, "repo", "base_commit", "parent_commit", "workdir", "language", + shown = {"problem_statement", "prompt", "system_prompt", "question", "info", "task", *IMAGE_COLS, "repo", "base_commit", "parent_commit", "workdir", "language", "repo_language", "version", "test_cmd", "install_config"} rest = {k: v for k, v in clean.items() if k not in shown and v not in (None, "", [], {})} if rest: @@ -361,10 +364,10 @@ class Verifiers(Processor): def run_blocks(spec: str, env: dict[str, Any], config: str, split: str, row: int | None = None) -> list[dict[str, Any]]: - """Commands that evaluate these rows with Verifiers 0.3.1 (the current release) and the Prime CLI.""" + """Commands targeting the release that exposes both the v0 and v1 CLIs.""" eid = env.get("env") head = ["# Verifiers (pip: verifiers 0.3.1) and the Prime CLI, in a new uv project", - "uv init vf-eval-here && cd vf-eval-here && uv add verifiers", "uv tool install prime"] + "uv init vf-eval-here && cd vf-eval-here && uv add 'verifiers==0.3.1'", "uv tool install prime"] if not eid: lines = head + ["", "# find and install the environment that loads this dataset (its card or repository names it)", "prime env list", "prime env install /", "", diff --git a/app/http.py b/app/http.py index dd0ad50ad47232dd30d68c6dd2f1998a2c688a25..0bd54b2036d2aa3030c19e4b9308b295873e4125 100644 --- a/app/http.py +++ b/app/http.py @@ -6,6 +6,7 @@ from __future__ import annotations import os import threading import time +from urllib.parse import urlsplit from fastapi import FastAPI, Request from fastapi.responses import JSONResponse @@ -21,7 +22,7 @@ CSP = ("default-src 'self'; script-src 'self' https://cdn.jsdelivr.net; style-sr def client_ip(request: Request) -> str: """The visitor's address: the last X-Forwarded-For entry is the one the Space's proxy added (earlier ones are whatever the client claimed).""" - fwd = [p.strip() for p in request.headers.get("x-forwarded-for", "").split(",") if p.strip()] + fwd = [p.strip() for p in request.headers.get("x-forwarded-for", "").split(",") if p.strip()] if config.TRUST_PROXY else [] return fwd[-1] if fwd else (request.client.host if request.client else "?") @@ -51,9 +52,22 @@ class Limiter: def same_origin(request: Request) -> bool: + """Compare complete origins; forwarding headers supplied by a client aren't authority.""" origin = request.headers.get("origin") - host = request.headers.get("x-forwarded-host") or request.headers.get("host") - return not origin or origin.split("://", 1)[-1] == host + if not origin: + return request.headers.get("sec-fetch-site") in (None, "same-origin", "none") + + def canonical(value: str): + try: + u = urlsplit(value) + if u.scheme not in ("http", "https") or not u.hostname or u.username or u.password or u.path or u.query or u.fragment: + return None + return u.scheme, u.hostname.lower(), u.port or (443 if u.scheme == "https" else 80) + except ValueError: + return None + + expected = canonical(config.PUBLIC_URL) if config.PUBLIC_URL else canonical(f"{request.url.scheme}://{request.headers.get('host', '')}") + return expected is not None and canonical(origin) == expected def install(app: FastAPI) -> None: @@ -61,7 +75,11 @@ def install(app: FastAPI) -> None: async def _revalidate(request: Request, call_next): """Code and styles revalidate on every load (cheap: ETag -> 304), so a deploy is never half-cached.""" resp = await call_next(request) - if not request.url.path.startswith("/api/"): + if request.url.path.startswith(("/api/", "/mcp/", "/login")): + # APIs can contain a visitor's private dataset, run, or account. A shared + # proxy or the browser's cache must never reuse them for another visitor. + resp.headers["Cache-Control"] = "private, no-store" + else: resp.headers.setdefault("Cache-Control", "no-cache") return resp @@ -88,10 +106,6 @@ def install(app: FastAPI) -> None: """The session cookie is SameSite=None (it has to work in the huggingface.co iframe), so a state-changing request must prove it came from this page: another site could otherwise start rollouts billed to you.""" if request.method not in ("GET", "HEAD", "OPTIONS") and request.url.path.startswith("/api/"): - origin = request.headers.get("origin") - host = request.headers.get("x-forwarded-host") or request.headers.get("host") - if origin and origin.split("://", 1)[-1] != host: - return JSONResponse({"detail": "cross-site request refused"}, 403) - if not origin and request.headers.get("sec-fetch-site") not in (None, "same-origin", "none"): + if not same_origin(request): return JSONResponse({"detail": "cross-site request refused"}, 403) return await call_next(request) diff --git a/app/indexer.py b/app/indexer.py index b4d7fb6df250d303334f66f7c6d514e23e22d99d..a4b5e26e41f3a34c597aec7f81f41f9ada0e32bd 100644 --- a/app/indexer.py +++ b/app/indexer.py @@ -231,6 +231,9 @@ def refresh_indexes(items: list[dict[str, Any]], state: dict[str, Any], *, budge th.join(timeout=max(60.0, deadline - time.time() + 120)) if th.is_alive(): out["abandoned"].append(spec) + # A bounded automatic worker exits after publication. Avoid retrying + # the same oversized build at the front of every subsequent batch. + state["failed"][spec] = {"sha": sha, "at": time.time(), "error": "Index build exceeded its time budget"} _ev("build_abandoned", spec=spec, seconds=round(time.time() - t0)) continue if "error" in result: @@ -318,7 +321,7 @@ def prune(store, keep: int) -> list[str]: def run(store, *, rows: list[dict[str, Any]] | None = None, budget_s: float = 1800, max_builds: int = 100, keep: int = 5, min_ratio: float = 0.8, force: bool = False, index: bool = True, dry_run: bool = False, - out: Path | None = None) -> dict[str, Any]: + out: Path | None = None, listing_at: float | None = None) -> dict[str, Any]: """One indexer run (see the module docstring). `rows` stands in for the Hub listing (tests). Raises IndexerError (or anything else) without publishing.""" from . import catalog, config, snapshot @@ -330,17 +333,18 @@ def run(store, *, rows: list[dict[str, Any]] | None = None, budget_s: float = 18 _ev("pointer_unreadable", error=str(e)) prev = None rows = rows if rows is not None else fetch_listing() + listing_at = listing_at if listing_at is not None else time.time() rows = [r for r in rows if not r.get("private") and not r.get("gated")] if not force: guard(rows, prev, min_ratio) # catalog's own readers (environments, featured lookups) see this listing, and the app's fallback path finds it with catalog._listing_lock: - catalog._env_list.update(rows=rows, at=time.time(), full=True) + catalog._env_list.update(rows=rows, at=listing_at, full=True) if not dry_run: import gzip catalog.atomic_write(config.STORAGE_DIR / "listing.json.gz", - gzip.compress(json.dumps({"at": time.time(), "full": True, "rows": rows}, default=str).encode())) + gzip.compress(json.dumps({"at": listing_at, "full": True, "rows": rows}, default=str).encode())) report: dict[str, Any] = {"listing": len(rows)} state = load_state() if index and not dry_run: diff --git a/app/main.py b/app/main.py index 6147d0182f9847d244429078ba89621a1b18dd85..e65a7af2d195083df040a8bd8dfe1aaa371b1992 100644 --- a/app/main.py +++ b/app/main.py @@ -23,6 +23,7 @@ from pydantic import BaseModel, Field from . import auth, catalog, config, endpoints, envs, http, mcp_bridge, models, runner, runtime, search_api, seo, settings, space_files, spaces_live, store, version from .envs import registry +from . import community as community_data app = FastAPI(title="HF RL Explorer", docs_url="/api/docs") app.include_router(auth.router) @@ -50,6 +51,10 @@ def _shutdown() -> None: """Live rollouts' events to the store, and locally the capture proxy's tunnels closed: gradio starts `frpc` as a child that outlives this process otherwise, and stale tunnels pile up across restarts.""" import sys + from . import auto_index, space_checks + + space_checks.stop() + auto_index.stop() store.flush_due(force=True) tunneling = sys.modules.get("gradio.tunneling") @@ -63,9 +68,12 @@ def _shutdown() -> None: @app.on_event("startup") def _startup() -> None: import threading + from . import auto_index, space_checks config.STORAGE_DIR.mkdir(parents=True, exist_ok=True) config.INDEX_DIR.mkdir(parents=True, exist_ok=True) + space_checks.start() + auto_index.start() # rollouts whose worker is gone are marked interrupted by the watcher below, once they've been silent a while threading.Thread(target=runner.watch_cancel_requests, daemon=True, name="cancel-requests").start() if os.environ.get("RLX_WARM", "1") == "1": @@ -157,8 +165,15 @@ def environments(): @app.get("/api/spaces/{org}/{name}") def space_page(org: str, name: str): - """An environment Space: its card and README, and what its OpenEnv server says about itself.""" - return _hub_errors(catalog.space, f"{org}/{name}") + """Repository metadata is distinct from verified live OpenEnv support.""" + from . import space_checks + source = _hub_errors(catalog.space, f"{org}/{name}") + check = space_checks.inventory().get(f"{org}/{name}", {}) + verified = space_checks.verified(check) + return {**source, "declared_openenv": bool(source.get("openenv")), "openenv": verified, + "framework": "openenv" if verified else "ors" if source.get("framework") == "ors" else "space", + "badges": [b for b in source.get("badges", []) if verified or b != "OpenEnv"], + "api_status": space_checks.status(check)} # ── live Spaces: status, wake/restart, Task API, playground ───────────────── @@ -181,7 +196,7 @@ def space_live(org: str, name: str, fresh: bool = False): with no MCP tools of its own (a reset/step environment) still has tools for an agent: `bridge_tools` are exactly what this explorer's MCP bridge offers for it (reset, step with its typed action, state, its Task API), for the page.""" info = _live(spaces_live.probe, f"{org}/{name}", fresh) - if info.get("running") and not info.get("mcp") and (info.get("step_api") or info.get("api") or info.get("task_api")): + if info.get("running") and (info.get("mcp") or info.get("step_api") or info.get("api") or info.get("task_api")): info = {**info, "bridge_tools": mcp_bridge._tools(info)} return info @@ -204,14 +219,14 @@ def space_restart(org: str, name: str, request: Request): @app.get("/api/spaces/{org}/{name}/tasks") -def space_tasks(org: str, name: str, split: str = "", start: int = 0, stop: int = 20): +def space_tasks(org: str, name: str, split: str = "", start: int = 0, stop: int = 20, env: str = ""): """A page of tasks from the server's Task API (answer-like fields left out).""" - return _live(spaces_live.tasks, f"{org}/{name}", split, start, stop) + return _live(spaces_live.tasks, f"{org}/{name}", split, start, stop, env) @app.get("/api/spaces/{org}/{name}/task") -def space_task(org: str, name: str, split: str = "", index: int = 0): - return _live(spaces_live.task, f"{org}/{name}", split, index) +def space_task(org: str, name: str, split: str = "", index: int = 0, env: str = ""): + return _live(spaces_live.task, f"{org}/{name}", split, index, env) class PlayIn(BaseModel): @@ -335,9 +350,10 @@ def env_data(org: str, name: str, request: Request, ref: str, part: str): def env_raw(org: str, name: str, request: Request, ref: str, f: str, download: bool = False): """One of a task's files as it is (an image, a page for a sandboxed frame). Never runs with this site's origin.""" data, media = _env(registry.raw, f"{org}/{name}", ref, f, _token(request)) - fname = f.rsplit("/", 1)[-1].replace('"', "") + # RFC 5987 handles Unicode and keeps control characters out of HTTP headers. + fname = quote(f.rsplit("/", 1)[-1], safe="") return Response(data, media_type=media, headers={"Content-Security-Policy": "sandbox", "X-Content-Type-Options": "nosniff", - "Content-Disposition": f'{"attachment" if download else "inline"}; filename="{fname}"'}) + "Content-Disposition": f"{'attachment' if download else 'inline'}; filename*=UTF-8''{fname}"}) @app.get("/api/env/{org}/{name}/random") @@ -484,19 +500,19 @@ PUBLIC_FIELDS = ("id", "dataset", "path", "task_id", "title", "collection", "dif def _shareable(r: dict) -> bool: """Shown to everyone: finished and graded, on a public dataset, and not taken off Community by an admin.""" - return (r.get("status") == "done" and r.get("reward") is not None and not r.get("restricted") - and r["id"] not in set(settings.get("hidden_runs", []))) + return community_data.is_public(r, set(settings.get("hidden_runs", []))) def _public_view(r: dict) -> dict: v = {k: r.get(k) for k in PUBLIC_FIELDS} + v["runner"] = community_data.runner(r) if r.get("endpoint"): v["endpoint"] = {"custom": True} return {**v, "visibility": "public"} def _owner_view(r: dict) -> dict: - return {**r, "is_owner": True} + return {**r, "runner": community_data.runner(r), "is_owner": True} def _visible(request: Request, run_id: str) -> tuple[dict, bool]: @@ -530,7 +546,7 @@ def my_runs(request: Request, d: str = "", p: str | None = None): @app.get("/api/runs/{run_id}") def run_detail(run_id: str, request: Request, after: int = 0): r, owner = _visible(request, run_id) - if r.get("runner") == "mimo": # the MiMo harness: its events, live while it runs (app/mimo/runner/core.py) + if community_data.runner(r) in ("mimo", "nemo-gym"): # event-based hosted runners from .mimo.runner import core as mimo live = mimo.live(run_id) @@ -570,85 +586,17 @@ def set_visibility(run_id: str, body: VisibilityIn, request: Request): return {"run": _owner_view(store.update(run_id, visibility=body.visibility))} -DOMAINS = {"code": "Code", "webdev": "Webdev", "cyber": "Cyber", "music": "Music", "general": "General"} - - -def _group(r: dict) -> str: - """Where a rollout's task belongs, for the leaderboard's columns: a MiMo domain, else its environment.""" - if r.get("domain") in DOMAINS: - return DOMAINS[r["domain"]] - return (r.get("dataset") or "?").split("/")[-1] - - -def _served(r: dict) -> str: - return "own endpoint" if r.get("endpoint") else (r.get("provider") or "auto") - - -def _band(r: dict) -> str: - rw = r.get("reward") - if rw is None or r.get("status") != "done": - return "unscored" - return "full" if rw >= 0.999 else "zero" if rw <= 0 else "partial" - - -def _stats(runs: list[dict]) -> dict: - """How the rollouts shown did: the spread of rewards, each model's mean, and each model in each group.""" - scored = [r for r in runs if r.get("reward") is not None and r.get("status") == "done"] - hist = [0] * 10 - for r in scored: - hist[min(9, max(0, int(float(r["reward"]) * 10)))] += 1 - by: dict[str, list[float]] = {} - for r in scored: - by.setdefault(r.get("model") or "custom endpoint", []).append(float(r["reward"])) - models_ = sorted(({"model": m, "custom": any(x.get("endpoint") for x in runs if (x.get("model") or "custom endpoint") == m), "runs": len(v), - "mean": round(sum(v) / len(v), 4), "best": max(v), "full": sum(x >= 0.999 for x in v), - "solved": sum(x > 0 for x in v)} for m, v in by.items()), key=lambda x: (-x["mean"], -x["runs"])) - cells: dict[str, dict[str, list[float]]] = {} - for r in scored: - cells.setdefault(r.get("model") or "custom endpoint", {}).setdefault(_group(r), []).append(float(r["reward"])) - matrix = {m: {g: {"mean": round(sum(v) / len(v), 4), "runs": len(v)} for g, v in gs.items()} for m, gs in cells.items()} - return {"runs": len(runs), "scored": len(scored), "mean": round(sum(float(r["reward"]) for r in scored) / len(scored), 4) if scored else None, - "full": sum(float(r["reward"]) >= 0.999 for r in scored), "histogram": hist, "by_model": models_, "models": models_, "matrix": matrix, - "datasets": len({r.get("dataset") for r in runs}), "tasks": len({(r.get("dataset"), r.get("path")) for r in runs})} - - -def _counts(runs: list[dict], f) -> list[list]: - c: dict[str, int] = {} - for r in runs: - v = f(r) - if v: - c[v] = c.get(v, 0) + 1 - return [[k, n] for k, n in sorted(c.items(), key=lambda kv: -kv[1])] - - @app.get("/api/community") def community(dataset: str = "", group: str = "", model: str = "", served: str = "", judge: str = "", reward: str = "", thinking: str = "", runner: str = "", q: str = "", sort: str = "new", d: str = "", p: str | None = None, limit: int = 100, offset: int = 0): - """Public, graded rollouts, by anyone, without who ran them: filtered every way a reader might want, and how each - model does, overall and in each environment (MiMo's domains count as environments of their own).""" + """Finished, graded public rollouts from any saved runner, without account identities or private endpoint locations.""" everyone = [r for r in store.list_runs(public=True, limit=100000) if _shareable(r)] - dataset = dataset or d - - def keep(r): - return ((not dataset or r.get("dataset") == dataset) and (p is None or r.get("path") == p) and (not group or _group(r) == group) - and (not model or (r.get("model") or "") == model) and (not served or _served(r) == served) - and (not judge or (r.get("judge") or "") == judge) and (not reward or _band(r) == reward) - and (not thinking or ((r.get("params") or {}).get("thinking") or "default") == thinking) - and (not runner or (r.get("runner") or "harbor") == runner) - and (not q or q.lower() in f"{r.get('title', '')} {r.get('task_id', '')} {r.get('path', '')}".lower())) - runs = [r for r in everyone if keep(r)] - key = {"reward_desc": lambda r: -(r.get("reward") if r.get("reward") is not None else -1), - "reward_asc": lambda r: (r.get("reward") if r.get("reward") is not None else 2), - "cost_asc": lambda r: (r.get("cost") or {}).get("total", 0), "cost_desc": lambda r: -(r.get("cost") or {}).get("total", 0)}.get(sort) - if key: - runs.sort(key=key) - facets = {"group": _counts(everyone, _group), "dataset": _counts(everyone, lambda r: r.get("dataset")), - "model": _counts(everyone, lambda r: r.get("model")), "served": _counts(everyone, _served), "judge": _counts(everyone, lambda r: r.get("judge")), - "reward": _counts(everyone, _band), "thinking": _counts(everyone, lambda r: (r.get("params") or {}).get("thinking") or "default"), - "runner": _counts(everyone, lambda r: r.get("runner") or "harbor")} - page = runs[max(0, offset): max(0, offset) + max(1, min(limit, 500))] - return {"stats": _stats(runs), "facets": facets, "total": len(runs), "tasks": len({(r.get("dataset"), r.get("path")) for r in runs}), - "runs": [{**_public_view(r), "group": _group(r), "shot": r.get("domain") == "webdev" and store.has_artifact(r["id"], "screenshot.jpg")} for r in page]} + filters = {"dataset": dataset or d, "path": p, "group": group, "model": model, "served": served, "judge": judge, + "reward": reward, "thinking": thinking, "runner": runner, "q": q[:200]} + out = community_data.query(everyone, filters, sort=sort, offset=offset, limit=limit) + out["runs"] = [{**_public_view(r), "runner": community_data.runner(r), "group": community_data.group(r), + "shot": r.get("domain") == "webdev" and store.has_artifact(r["id"], "screenshot.jpg")} for r in out["runs"]] + return out @app.get("/api/community/tasks") @@ -664,7 +612,7 @@ def community_tasks(dataset: str): if r.get("reward") is not None: t["scored"] += 1 t["sum"] += float(r["reward"]) - t["best"] = max(t["best"] if t["best"] is not None else 0, float(r["reward"])) + t["best"] = max(t["best"], float(r["reward"])) if t["best"] is not None else float(r["reward"]) return {"tasks": {k: {"runs": v["runs"], "mean": round(v["sum"] / v["scored"], 4) if v["scored"] else None, "best": v["best"], "last": v["last"], "title": v["title"]} for k, v in out.items()}} @@ -692,6 +640,14 @@ def run_artifact(run_id: str, name: str, request: Request): LLM_REPLY_MAX = 64 * 1024 * 1024 +@app.get("/api/llm/{cap}/health") +def llm_health(cap: str): + from .mimo.runner import core as events + if events.by_cap(cap) is None: + raise HTTPException(404, "unknown or finished rollout") + return {"status": "ok"} + + @app.post("/api/llm/{cap}/v1/chat/completions") async def llm_proxy(cap: str, request: Request): import asyncio @@ -712,6 +668,8 @@ async def llm_proxy(cap: str, request: Request): body = json.loads(raw) except ValueError: return JSONResponse({"error": {"message": "invalid JSON"}}, 400) + if not isinstance(body, dict): + return JSONResponse({"error": {"message": "JSON object required"}}, 400) up = r.upstream(str(body.get("model") or "")) if up is None: return JSONResponse({"error": {"message": "this rollout may not call that model"}}, 403) diff --git a/app/mcp_bridge.py b/app/mcp_bridge.py index a6fdc5208708c037ef0627cf6e11a191261fa29c..884c5af5a560d6c14d37069aef0e62d98c1328ae 100644 --- a/app/mcp_bridge.py +++ b/app/mcp_bridge.py @@ -51,6 +51,9 @@ def _limited(owner: str) -> bool: now = time.time() with _lock: hits = [t for t in _hits.get(owner, []) if now - t < RATE[1]] + if len(hits) >= RATE[0]: + _hits[owner] = hits + return True hits.append(now) _hits[owner] = hits if len(_hits) > 5000: @@ -84,19 +87,31 @@ def _tools(info: dict) -> list[dict[str, Any]]: ] # a tool server takes its actions as tool calls: it gets reset and state, not step tools += [t for t in episode if not (names and t["name"] == "step")] - # a server's own routes (NeMo Gym, ORS, custom): one tool each, named after its path, when it has no tools of its own - if not info.get("mcp") and info.get("api"): + # Keep published grading/seed routes alongside native tools, without naming collisions. + if info.get("api"): taken = {t["name"] for t in tools} for r in info["api"]: name = _route_tool(r) if name in taken: continue taken.add(name) - props = {p: {"type": "string", "description": "in the path"} for p in r["params"]} - body = r.get("schema") or {} - props.update({k: v for k, v in (body.get("properties") or {}).items()}) + # Separate request locations: a body key named 'id' or 'query' must + # not overwrite a path/query parameter with the same name. + props, required = {}, [] + if r["params"]: + props["path_params"] = {"type": "object", "properties": {p: {"type": "string"} for p in r["params"]}, + "required": r["params"], "additionalProperties": False} + required.append("path_params") + if (r.get("query_schema") or {}).get("properties"): + props["query"] = r["query_schema"] + if r["query_schema"].get("required"): + required.append("query") + if r["method"] == "POST": + props["body"] = r.get("schema") or {"type": "object"} + if r.get("body_required"): + required.append("body") tools.append({"name": name, "description": f"{r['method']} {r['path']}" + (f": {r['summary']}" if r["summary"] else ""), - "inputSchema": {"type": "object", "properties": props, "required": list(r["params"]) + list(body.get("required") or [])}}) + "inputSchema": {"type": "object", "properties": props, "required": required, "additionalProperties": False}}) if info.get("task_api"): extra = [ {"name": "list_splits", "description": "The server's task splits and how many tasks each holds.", "inputSchema": {"type": "object", "properties": {}}}, @@ -163,11 +178,12 @@ def _call(spec: str, info: dict, client: dict, name: str, args: dict) -> dict[st if name == "list_tasks": return _content(live.tasks(spec, str(args.get("split", "")), int(args.get("start", 0) or 0), int(args.get("stop", 20) or 20))) return _content(live.task(spec, str(args.get("split", "")), int(args.get("index", 0) or 0))) - route = next((r for r in info.get("api") or [] if _route_tool(r) == name), None) if not own else None + route = next((r for r in info.get("api") or [] if _route_tool(r) == name), None) if route is not None: - params = {p: args.get(p) for p in route["params"]} - body = {k: v for k, v in args.items() if k not in route["params"]} if route["method"] == "POST" else None - out = live.act(_play(spec, client), client["owner"], "http", {"method": route["method"], "path": route["path"], "params": params, "body": body})["result"] + params = args.get("path_params") if isinstance(args.get("path_params"), dict) else {} + query = args.get("query") if isinstance(args.get("query"), dict) else {} + body = args.get("body") if route["method"] == "POST" else None + out = live.act(_play(spec, client), client["owner"], "http", {"method": route["method"], "path": route["path"], "params": params, "query": query, "body": body})["result"] payload = out.get("json", out.get("text")) result = _content(payload if payload is not None else {"status": out["status"]}) if out["status"] >= 400: diff --git a/app/mimo/__init__.py b/app/mimo/__init__.py index 7c0f3c3b64cdfb0fcd181531bb147ab2eb54a34d..46eafc3fd1e258d835d5cc6d64456ae03c5d8aa6 100644 --- a/app/mimo/__init__.py +++ b/app/mimo/__init__.py @@ -3,7 +3,7 @@ every one of its 7,780 tasks browsable in full (brief, agent prompt, systems and repository, the grader with its judge prompts), and rollouts on its own harness: OpenCode on an HF Sandbox, graded by Xiaomi's graders (hidden tests, a fuzz crash, rubric checks with a judge model, a screenshot judge, the music scorer). -The code is the MiMo explorer's (mimo-explorer/app), kept close to it so the two can be compared and synced: +The code is the MiMo explorer's (tooling/mimo-rl-explorer/app), kept close to it so the two can be compared and synced: catalog.py the task views and what the runner needs (data/: the browsing index, built by build_data.py) judges.py the judge prompts, pinned previews.py workspace file previews (spreadsheets, documents, slides, PDFs) diff --git a/app/mimo/runner/core.py b/app/mimo/runner/core.py index 851290055a59b165fa6b7fcfbc6ee429c559e929..12112ad0aa301704691c7d9223ea7f38927f000f 100644 --- a/app/mimo/runner/core.py +++ b/app/mimo/runner/core.py @@ -137,7 +137,7 @@ class Rollout: self.update(flavor=flavor) t = time.time() self.sandbox = Sandbox.create(image=image, flavor=flavor, idle_timeout=config.SANDBOX_IDLE_TIMEOUT, - forward_hf_token=not config.PUBLIC_URL, token=self.token, start_timeout=900, + forward_hf_token=False, token=self.token, start_timeout=900, labels={"app": "rl-explorer", "runner": "mimo", "run": self.id}) self.update(sandbox_id=self.sandbox.id) self.phase("sandbox", "done", f"ready in {time.time() - t:.0f}s") @@ -152,8 +152,30 @@ class Rollout: return config.ROUTER, self.token, mid def llm_base(self) -> str | None: - """The OpenAI-compatible base URL the sandbox should use, or None to call the provider directly.""" - return f"{config.PUBLIC_URL}/api/llm/{self.cap}/v1" if config.PUBLIC_URL else None + """Only a per-run capability reaches the sandbox, including local previews.""" + if config.PUBLIC_URL: + return f"{config.PUBLIC_URL}/api/llm/{self.cap}/v1" + from ...runner import _proxy_url + return f"{_proxy_url().rstrip('/')}/rlx/llm/{self.cap}/v1" + + def check_model_network(self): + """Check the exact relay path from the sandbox, without a model call.""" + url = self.llm_base().removesuffix("/v1") + "/health" + script = ("import urllib.request,urllib.error\n" + f"url={url!r}\n" + "try:\n" + " r=urllib.request.urlopen(url,timeout=15)\n" + " print('relay HTTP',r.status)\n" + "except urllib.error.HTTPError as e:\n" + " print('relay HTTP',e.code);raise SystemExit(1)\n" + "except Exception as e:\n" + " print(type(e).__name__);raise SystemExit(1)\n") + import shlex + result = self.sh("python3 -c " + shlex.quote(script), timeout=25) + if result.exit_code: + raise RuntimeError("The sandbox cannot reach the model relay. Check HTTPS egress from the sandbox and ingress to the explorer. " + "No account token was forwarded. " + (result.stdout or "")[-120:]) + self.log("Network check passed: sandbox → model relay. HF account token stays on the explorer.") def upstream(self, model: str) -> tuple[str, str | None] | None: """For the model proxy: (base URL, key) for a model this rollout may call, else None.""" @@ -193,12 +215,16 @@ class Rollout: def execute(self) -> None: from .domains import ADAPTERS - adapter = ADAPTERS[self.run["domain"]] + if self.run.get("runner") == "nemo-gym": + from ... import nemo_runner as adapter + else: + adapter = ADAPTERS[self.run["domain"]] try: self.update(status="starting", started_at=time.time()) self.settle_model() result = adapter.run(self) - self.update(status="done", reward=result.get("reward"), reward_error=result.get("error"), + self.update(status="done", phase="done", reward=result.get("reward"), reward_error=result.get("error"), + graded=result.get("reward") is not None, wall_s=round(time.time() - self.t0, 2), finished_at=time.time(), cost=self.cost_now()) self.phase("done", "done", f"reward {result.get('reward')}" if result.get("reward") is not None else f"not scored: {result.get('error')}") @@ -217,10 +243,12 @@ class Rollout: self._flush() self.stop_sandbox() self.agent_key = None + self.native_task = None with _live_lock: _caps.pop(self.cap, None) with _live_lock: _live.pop(self.id, None) + self.token = "" def stop_sandbox(self) -> None: if self.sandbox is None: @@ -270,11 +298,12 @@ def submit(user: str, token: str, task: dict, model: str, provider: str | None, "facets": task.get("facets"), "model": model, "provider": provider, "judge": judge, "status": "queued", "harness": "opencode", "endpoint": endpoint, "params": params or {}, "visibility": "private" if visibility == "private" else "public", - "image": catalog.image_for(task["id"]) if task["domain"] != "music" else None, + "image": catalog.image_for(task["id"]) if task["domain"] not in ("music", "nemo-gym") else None, "provenance": version.provenance(), **(extra or {}), }) r = Rollout(run, token, agent_key) + r.native_task = task.get("native_task") # reference action stays in memory; never in the run record with _live_lock: _live[r.id] = r _caps[r.cap] = r diff --git a/app/mimo/runner/opencode.py b/app/mimo/runner/opencode.py index 0d6ed4baaaa619c161e52245a981c7a47015f67b..c7a19c6a3351e9f702a26aa779ef320c056b01c2 100644 --- a/app/mimo/runner/opencode.py +++ b/app/mimo/runner/opencode.py @@ -119,6 +119,7 @@ def run(r, prompt: str, cwd: str, *, steps: int, timeout: float, user: str | Non steps = params.get("steps") or steps timeout = params["timeout_min"] * 60 if params.get("timeout_min") else timeout proxy = r.llm_base() + r.check_model_network() cfg = config_for(r.run["model"], r.run.get("provider"), steps, mcp, ep, params, proxy) r.sandbox.files.write("/tmp/opencode.json", json.dumps(cfg)) r.sandbox.files.write("/tmp/prompt.txt", prompt) @@ -166,6 +167,10 @@ def run(r, prompt: str, cwd: str, *, steps: int, timeout: float, user: str | Non r.sandbox.run("rm -f /root/.agent_key", shell=True, check=False, timeout=30) if buf[0].strip(): _event(r, buf[0], texts) + if getattr(r, "agent_failure", None): + raise RuntimeError("The agent's model call failed; this run is not graded. " + r.agent_failure[:500]) + if res.exit_code and not res.timed_out: + raise RuntimeError(f"The agent exited with code {res.exit_code}; this run is not graded.") if res.timed_out: r.emit("error", text=f"The agent hit the {int(timeout // 60)}-minute limit; grading what it left.") return texts[-1] if texts else "" @@ -209,8 +214,8 @@ def _event(r, line: str, texts: list[str]) -> None: "so OpenCode ends the session there, as it does in Xiaomi's harness.") elif typ == "error": err = ev.get("error") or part - r.emit("error", text=(err.get("data") or {}).get("message") or json.dumps(err)[:800] - if isinstance(err, dict) else str(err)[:800]) + r.agent_failure = ((err.get("data") or {}).get("message") or json.dumps(err)[:800]) if isinstance(err, dict) else str(err)[:800] + r.emit("error", text=r.agent_failure) def _scripted(r, cwd: str, user: str | None) -> str: diff --git a/app/nemo_runner.py b/app/nemo_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..81f04c4467f5c74036a0301ffebc5883b4e413a6 --- /dev/null +++ b/app/nemo_runner.py @@ -0,0 +1,158 @@ +"""One-step NeMo Pivot prediction, scored by NVIDIA's unmodified comparator. + +Tool definitions are prediction targets; tool calls are never executed. No task +code, model key, or reference action enters a sandbox. Other Gym agents require +their resources server and are intentionally outside this adapter's scope. +""" +from __future__ import annotations + +import json + +import httpx +from pydantic import TypeAdapter + +from .vendor.nemo_gym.verification_utils import ( + ActionComparator, ExpectedAction, FunctionCallAction, FunctionCallBatchAction, + MessageAction, StepRewardCategory, ToolCallComparatorConfig, +) + +REVISION = "3ef478df1ee163134d32a3f291f0a9e5981d0e52" +CONFIGS = { + "resources_servers/single_step_tool_use_with_argument_comparison/configs/" + + name + ".yaml" for name in ( + "single_step_tool_use_with_argument_comparison", + "toolcall_schema_single_step_tool_use_with_argument_comparison", + ) +} + + +def supported(agent): + return agent.get("known") and agent.get("agent_type") == "tool_simulation_agent" and agent.get("config") in CONFIGS + + +def strict_schema(schema): + if not isinstance(schema, dict): + return False + if schema.get("type") == "object" or "properties" in schema: + if schema.get("additionalProperties") is not False or set(schema.get("required") or []) != set(schema.get("properties") or {}): + return False + return all(strict_schema(v) for k, v in schema.items() if k in ("items", "additionalProperties") and isinstance(v, dict)) and all( + strict_schema(v) for v in (schema.get("properties") or {}).values()) and all( + strict_schema(v) for k in ("anyOf", "oneOf", "allOf") for v in schema.get(k, [])) + + +def relaxed_tools(row): + return [t["name"] for t in (row.get("responses_create_params") or {}).get("tools", []) + if isinstance(t, dict) and t.get("name") and t.get("strict") is True and not strict_schema(t.get("parameters"))] + + +def responses_request(row): + """Keep NeMo's native Responses input, including prior reasoning and tool turns. + + Only function definitions are allowed: provider-hosted MCP, web search and + code execution would change a prediction task into a side-effecting agent. + """ + rcp = row.get("responses_create_params") + if not isinstance(rcp, dict) or not rcp.get("input"): + raise ValueError("This row has no NeMo Gym Responses request") + if len(json.dumps(rcp)) > 200_000: + raise ValueError("This task's model input exceeds the hosted limit") + raw = rcp["input"] + if not isinstance(raw, (str, list)) or isinstance(raw, list) and len(raw) > 500: + raise ValueError("This row needs the native NeMo Gym runtime") + tools = rcp.get("tools") or [] + if not isinstance(tools, list) or len(tools) > 128 or not all( + isinstance(t, dict) and t.get("type") == "function" and isinstance(t.get("name"), str) for t in tools + ): + raise ValueError("Only function prediction tools are supported here; use the native NeMo Gym runtime") + allowed = {"input", "instructions", "tools", "tool_choice", "temperature", "top_p", "parallel_tool_calls", "reasoning", "text"} + # Model, token limits, persistence and execution mode belong to the runner. + body = {k: v for k, v in rcp.items() if k in allowed and v is not None} + relaxed = set(relaxed_tools(row)) + if tools: + body["tools"] = [{**t, "strict": False} if t["name"] in relaxed else t for t in tools] + return body + + +def prediction(response): + """Match NeMo's response_utils.extract_action: calls take precedence over text.""" + if response.get("error") or response.get("status") in ("failed", "cancelled"): + error = response.get("error") or {} + raise ValueError("Inference provider did not produce a prediction: " + str(error.get("message") or response.get("status"))[:600]) + if not isinstance(response.get("output"), list): + raise ValueError("Inference provider returned an invalid Responses payload") + if response.get("status") == "incomplete" and not response["output"]: + raise ValueError("The model reached its output limit without a prediction. Increase Max output tokens.") + calls, content = [], None + for item in response.get("output") or []: + if item.get("type") == "function_call": + calls.append({"function": {"name": item["name"], "arguments": item["arguments"]}}) + elif item.get("type") == "message" and item.get("role") == "assistant" and content is None: + content = next((p["text"] for p in item.get("content", []) if p.get("type") == "output_text"), None) + return {"tool_calls": calls, "content": content} + + +def validate(row, agent): + if not supported(agent): + raise ValueError("This NeMo resources server is not hosted here") + responses_request(row) + if len(json.dumps(row.get("expected_action"))) > 100_000: + raise ValueError("The reference action exceeds the hosted verifier limit") + try: + return TypeAdapter(ExpectedAction).validate_python(row.get("expected_action")) + except ValueError: + raise ValueError("The reference action is invalid for this NeMo Gym verifier") from None + + +def score(expected, message): + calls = message.get("tool_calls") or [] + if len(calls) > 64: + raise ValueError("The provider returned too many tool calls") + actions = [FunctionCallAction(type="function_call", name=c["function"]["name"], arguments=c["function"]["arguments"]) for c in calls] + actual = (actions[0] if len(actions) == 1 else FunctionCallBatchAction(type="function_call_batch", calls=actions)) if actions else ( + MessageAction(type="message", content=message["content"]) if isinstance(message.get("content"), str) else None) + if actual is None: + return 0.0, str(StepRewardCategory.NO_ACTION_FOUND) + result = ActionComparator(config=ToolCallComparatorConfig(word_count_similarity_threshold=0.1)).compare_action(expected, actual) + return result.reward, str(result.category) + + +def run(r): + payload = r.native_task + expected = validate(payload["row"], payload["agent"]) + body = responses_request(payload["row"]) + base, key, model = r.agent_api() + params = r.run["params"] + body.update(model=model, max_output_tokens=params.get("max_tokens", 2048), stream=False, store=False) + if params.get("temperature") is not None: + body["temperature"] = params["temperature"] + r.check_cancel() + r.phase("agent", detail="One prediction; tools are not executed") + if relaxed_tools(payload["row"]): + r.log("Some source tools declare strict=true with incompatible parameter schemas. Kept the original parameters and disabled strict enforcement for: " + + ", ".join(relaxed_tools(payload["row"]))) + r.emit("prompt", harness="nemo-gym", text=json.dumps({k: v for k, v in body.items() if k != "model"}, ensure_ascii=False, indent=2)) + response = httpx.post(base + "/responses", json=body, headers={"Authorization": f"Bearer {key}"}, timeout=90, follow_redirects=False) + if response.status_code != 200: + raise RuntimeError(f"Inference provider returned HTTP {response.status_code}; check model availability and Inference Providers permission") + r.check_cancel() + if len(response.content) > 2_000_000: + raise ValueError("The model response exceeds the hosted limit") + result = response.json() + message = prediction(result) + if message.get("content"): + r.emit("text", text=message["content"]) + for call in (message.get("tool_calls") or [])[:64]: + r.emit("tool", tool=call["function"]["name"], title="Predicted tool call", input=call["function"]["arguments"], + output="Prediction only; no tool was executed.", status="done") + usage = result.get("usage") or {} + r.add_tokens({"input": usage.get("input_tokens", 0), "output": usage.get("output_tokens", 0)}) + r.emit("step", tokens=r.tokens, cost=r.model_cost) + r.update(n_turns=1) + r.phase("agent", "done") + r.phase("verify", detail="NeMo Gym's pinned argument comparator") + reward, category = score(expected, message) + r.emit("checks", checks=[{"id": "native-comparator", "passed": reward == 1, "score": reward, + "method": "NeMo Gym", "message": category}], reward=reward, summary=category) + r.phase("verify", "done", category) + return {"reward": reward} diff --git a/app/runner.py b/app/runner.py index 9ab2a688e311b09e8f405f013ab0127122c48109..408db59fc10020c8ddb7b6727f36543fda22123b 100644 --- a/app/runner.py +++ b/app/runner.py @@ -20,6 +20,7 @@ import asyncio import contextlib import contextvars import json +import math import logging import re import secrets @@ -28,6 +29,7 @@ import threading import time from pathlib import Path from typing import Any +from fastapi import Request from . import catalog, config, dockerfile, judge, store @@ -243,6 +245,16 @@ async def _replay(env: Any, plan: Any) -> None: # ── the capture proxy ──────────────────────────────────────────────────────── +async def _event_model_proxy(cap: str, request: Request): + from .main import llm_proxy + return await llm_proxy(cap, request) + + +def _event_model_health(cap: str): + from .main import llm_health + return llm_health(cap) + + def service() -> Any: """OpenEnv's Harbor service: the capture proxy, and the URL the sandbox reaches it at.""" global _service @@ -253,6 +265,12 @@ def service() -> Any: # No default engine: every rollout brings its own (the visitor's token or endpoint). _service = HarborService(llm_url=config.ROUTER, model="", datasets=[], provider="hf", capture_level="text", expose=config.CAPTURE_EXPOSE) + _service.capture.app.add_api_route("/rlx/llm/{cap}/v1/chat/completions", _event_model_proxy, methods=["POST"]) + _service.capture.app.add_api_route("/rlx/llm/{cap}/health", _event_model_health, methods=["GET"]) + # OpenEnv ends with a catch-all POST capture route. Specific relay + # routes must precede it; they keep their own per-run capability auth. + routes = _service.capture.app.router.routes + routes[:] = routes[-2:] + routes[:-2] _patch_hf_sandbox() _patch_harbor_env() _patch_opencode_install() @@ -525,10 +543,13 @@ def submit(user: dict[str, Any], dataset: str, path: str, harness: str, model: s """Start a rollout of task `path` (the environment's ref) of `dataset`, with agent `harness`, by `runner`; `fields` are the run option's own inputs (contract.run_option fields), checked against it.""" from . import settings + from .envs.capabilities import HOSTED_RUNNERS token = user.get("token") if not token: raise PermissionError("sign in to run a rollout") + if runner not in HOSTED_RUNNERS: + raise ValueError(f"The {runner} runtime is not hosted here. Use the task's native framework instructions.") if not settings.get("rollouts_enabled", True): raise RuntimeError("Rollouts are paused for maintenance. Try again later.") if harness not in settings.get("agents", list(AGENT_IDS)): @@ -547,6 +568,18 @@ def submit(user: dict[str, Any], dataset: str, path: str, harness: str, model: s if endpoint and not opt.get("endpoint", True): raise ValueError(f"the {opt['label']} runner can't use your own endpoint") fields = _check_fields(opt.get("fields") or [], fields or {}) + if runner == "nemo-gym": + from . import nemo_runner + from .mimo.runner import core as events + with _slot(user["name"], max_user, max_all): + return events.submit(user["name"], token, + {"id": path, "domain": "nemo-gym", "title": row["title"], "native_task": row["native_task"]}, + model, provider, None, params=fields, + visibility="private" if row.get("restricted") else visibility, + extra={"dataset": env.id, "path": path, "task_id": f"{env.id}:{path}", "env": env.key, "adapter": env.adapter.id, + "runner": runner, "harness": "single-prediction", "restricted": bool(row.get("restricted")), + "sha": row.get("sha"), "provenance": {"framework": "NeMo Gym", "commit": nemo_runner.REVISION, + "source": "https://github.com/NVIDIA-NeMo/Gym/tree/" + nemo_runner.REVISION}}, reserved=True) if runner == "mimo": return _submit_mimo(user, env, row, model, provider, endpoint, endpoint_key, fields, visibility, max_user, max_all) params = {**params, **{k: fields[k] for k in ("steps", "timeout_min") if k in fields}} # the Harbor runner's limits @@ -623,8 +656,10 @@ def _check_fields(spec: list[dict[str, Any]], given: dict[str, Any]) -> dict[str if not isinstance(v, str) or v not in {m["id"] for m in models.catalog().get(f.get("pool") or "", [])}: raise ValueError(f"{f.get('label', key)}: pick one of the models offered") - if kind == "number" and (not isinstance(v, (int, float)) or isinstance(v, bool)): + if kind == "number" and (not isinstance(v, (int, float)) or isinstance(v, bool) or not math.isfinite(v)): raise ValueError(f"{f.get('label', key)} should be a number") + if kind == "number" and f.get("step") == 1 and v != int(v): + raise ValueError(f"{f.get('label', key)} should be a whole number") if kind == "number" and ((f.get("min") is not None and v < f["min"]) or (f.get("max") is not None and v > f["max"])): raise ValueError(f"{f.get('label', key)} should be between {f.get('min')} and {f.get('max')}") if kind == "bool" and not isinstance(v, bool): diff --git a/app/runtime.py b/app/runtime.py index 97fb75c57eb177bfc44005432149e41630432346..19f472b78185ea6385d92e0dee3fc0f25b6d8958 100644 --- a/app/runtime.py +++ b/app/runtime.py @@ -52,7 +52,7 @@ def more_threads() -> None: # ── logs ───────────────────────────────────────────────────────────────────── -SECRET_PATH = re.compile(r"(/api/llm/)[^/\s]+|((?:session|cap|token)=)[^&\s]+", re.I) +SECRET_PATH = re.compile(r"(/(?:api|rlx)/llm/)[^/\s]+|((?:session|cap|token)=)[^&\s]+", re.I) class _Json(logging.Formatter): diff --git a/app/search_api.py b/app/search_api.py index da16e9483088248ad6bee31ae8407a14a7499a8f..77fcd8b7cb9f7d94ea5efeabd8dbdc2eb3967957 100644 --- a/app/search_api.py +++ b/app/search_api.py @@ -5,14 +5,15 @@ /api/search answers exactly as the Explore page always filtered its full listing in the browser: - kind what an environment is: OpenEnv Spaces (a manifest or the tag, and FineEnvs' curated servers), other + kind OpenEnv Spaces require fresh successful API checks. Metadata-only matches are unverified-space; other environment Spaces (ORS included), Harbor datasets (tagged, or an index found task folders), Verifiers, NeMo Gym, OpenEnv datasets, verl and SkyRL, other RL datasets + owner exact Hub namespace, case-insensitive; applies to datasets and Spaces collection an admin's collection, "other" (in none) or "mine" (the signed-in visitor's datasets, with mine=1) f facet filters as the page's own URL writes them: `type:Benchmark|Neither;tags:code` (values URI-encoded); values of one facet are alternatives, facets all apply q every word must be part of the id, heading, brief or tags (substrings, any case) - sort trending (pinned first; with kind=openenv, FineEnvs' curated servers next) | downloads | likes | tasks | + sort trending (Hub score, then likes, then monthly dataset downloads; no editorial boosts) | downloads | likes | tasks | rollouts | updated | new; ties keep the listing's order Each facet is counted over what every *other* filter leaves (so picking a value never hides its alternatives). @@ -31,20 +32,24 @@ import re import threading import time from collections import OrderedDict -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any from urllib.parse import unquote from fastapi import APIRouter, HTTPException, Request from fastapi.responses import Response -from . import snapshot +from . import snapshot, space_checks router = APIRouter(on_startup=[snapshot.start]) -KINDS = ("all", "harbor", "openenv", "other-space", "verifiers", "nemo-gym", "openenv-data", "verl", "rows") +KINDS = ("all", "harbor", "openenv", "unverified-space", "other-space", "verifiers", "nemo-gym", "openenv-data", "verl", "rows") SORTS = ("trending", "downloads", "likes", "tasks", "rollouts", "updated", "new") -FACETS = ("type", "size", "stage", "mcp", "oe", "tags") +FACETS = ("type", "size", "health", "mode", "tools", "stage", "mcp", "oe", "oe_source", "evidence", "tags") +RANKING = {"version": 2, "source": "Hugging Face Hub", "order": ["trending", "likes", "downloads"], + "description": "Hub trending score first; ties use total likes, then dataset downloads over the last month. " + "Spaces have no download metric. Remaining ties keep catalog order. Featured picks do not affect rank.", + "scope": "Current search and filters", "quality": "Interest signals, not measured environment quality or live availability."} OLD_KINDS = {"dataset": "harbor", "space": "openenv", "ors": "other-space"} # the page's older links FACET_LIMIT = 100 # values returned per facet (most common first); the rest are counted in facets_more MAX_WORDS = 16 @@ -55,7 +60,9 @@ MAX_VALUES = 50 @dataclass class Query: q: str = "" + scope: str = "all" kind: str = "all" + owner: str | None = None coll: str | None = None sel: dict[str, list[str]] = field(default_factory=dict) sort: str = "trending" @@ -67,7 +74,7 @@ class Query: return self.q.lower().split()[:MAX_WORDS] def public(self) -> dict[str, Any]: - return {"q": self.q, "kind": self.kind, "collection": self.coll, "f": self.sel, "sort": self.sort, + return {"q": self.q, "scope": self.scope, "owner": self.owner, "kind": self.kind, "collection": self.coll, "f": self.sel, "sort": self.sort, "page": self.page, "size": self.size} @@ -79,6 +86,10 @@ def parse_f(f: str) -> dict[str, list[str]]: if k not in FACETS or not v: continue vals = [unquote(x)[:200] for x in v.split("|") if x] + if k == "stage" and "Asleep or stopped" in vals: # saved links from the former broad bucket + vals = [x for x in vals if x != "Asleep or stopped"] + ["Sleeping", "Paused", "Building", "Starting", "Error", "Deleting"] + if k == "mcp": + vals = ["MCP tagged" if x == "Has MCP tools" else x for x in vals] for x in vals: if x not in sel.setdefault(k, []) and len(sel[k]) < MAX_VALUES: sel[k].append(x) @@ -101,7 +112,7 @@ def parse_query(params: Any, collections: list[dict[str, Any]], mine: bool) -> Q coll = g("collection", "c") known = {c["id"] for c in collections} | {"other"} | ({"mine"} if mine else set()) sort = g("sort", "s") or "trending" - return Query(q=(g("q") or "").strip()[:200], kind=kind if kind in KINDS else "all", coll=coll if coll in known else None, + return Query(q=(g("q") or "").strip()[:200], scope="ready" if g("scope") == "ready" else "all", owner=(g("owner") or "").strip()[:200] or None, kind=kind if kind in KINDS else "all", coll=coll if coll in known else None, sel=parse_f(g("f") or ""), sort=sort if sort in SORTS else "trending", page=_int(g("page"), 1, 1, 10_000), size=_int(g("size"), 40, 1, 100)) @@ -155,12 +166,14 @@ class Ctx: hidden: list[str] mine: list[str] rolls: dict[str, int] + checks: dict[str, dict] = field(default_factory=dict) def params(self) -> dict[str, str]: """The admin settings as query parameters: each collection's keys, pins, hidden ones, the visitor's, rollouts. (Lists checked with IN, which SQLite evaluates once per statement: no per-row JSON.)""" out = {"pins": json.dumps(self.pins), "hidden": json.dumps(self.hidden), "mine": json.dumps(self.mine), - "rolls": json.dumps(self.rolls)} + "rolls": json.dumps(self.rolls), "checks": json.dumps([ + {**r, "status": space_checks.status(r), "browseable": space_checks.browseable(r)} for r in self.checks.values()])} for i, g in enumerate(self.collections): out[f"c{i}"] = json.dumps([k for k, c in self.cmap.items() if c == g["id"]]) out[f"cid{i}"] = g["id"] @@ -181,32 +194,53 @@ def context(mine: list[str] | None = None) -> Ctx: for k in g.get("ids") or []: cmap.setdefault(k, g["id"]) # the first collection naming it, as catalog.collection_of return Ctx(collections=[{k: v for k, v in g.items() if k != "ids"} for g in colls], cmap=cmap, pins=list(catalog.pinned()), - hidden=sorted(catalog.hidden()), mine=list(mine or []), rolls=rollout_counts()) + hidden=sorted(catalog.hidden()), mine=list(mine or []), rolls=rollout_counts(), checks=space_checks.inventory()) # ── SQL over the snapshot ──────────────────────────────────────────────────── BASE = """ -WITH b0 AS ( +WITH checks AS MATERIALIZED ( + SELECT CAST(json_extract(value, '$.id') AS TEXT) AS id, value AS check_json FROM json_each(:checks) +), b0 AS ( SELECT e.rowid AS rid, e.ord, e.id, e.key, e.kind, e.framework, e.openenv, e.heading, e.brief, e.downloads, e.likes, e.trending, e.created, e.updated, e.created_ms, e.updated_ms, e.stage, e.mcp, e.openenv_version, e.manifest, e.hardware, e.badges, e.tags, e.tasks, e.indexed, e.size_f, e.stage_f, e.mcp_f, e.oe_f, e.blob, + c.check_json, {coll} AS coll, (e.key IN (SELECT value FROM json_each(:pins))) AS pinned, (e.key IN (SELECT value FROM json_each(:mine))) AS mine FROM envs e + LEFT JOIN checks c ON c.id = e.id AND e.kind = 'space' WHERE e.key NOT IN (SELECT value FROM json_each(:hidden)) ), -b1 AS (SELECT b0.*, (kind = 'space' AND coll IS 'fineenvs' AND framework IS NOT 'openenv') AS ov FROM b0), base AS ( - SELECT b1.*, - CASE WHEN ov THEN 'openenv' ELSE framework END AS framework_q, - CASE WHEN ov THEN 1 ELSE openenv END AS openenv_q, - CASE WHEN ov THEN '["OpenEnv"]' ELSE badges END AS badges_q, - CASE WHEN kind = 'space' THEN (CASE WHEN ov OR framework IS 'openenv' OR openenv THEN 'openenv' ELSE 'other-space' END) - ELSE COALESCE(framework, 'harbor') END AS fw - FROM b1 + SELECT b0.*, + CASE WHEN kind = 'space' THEN CASE WHEN {verified} THEN 'openenv' WHEN framework = 'ors' THEN 'ors' ELSE 'space' END ELSE framework END AS framework_q, + CASE WHEN kind = 'space' AND {verified} THEN 1 ELSE 0 END AS openenv_q, + CASE WHEN kind = 'space' AND NOT COALESCE(({verified}), 0) THEN + (SELECT json_group_array(value) FROM json_each(badges) WHERE value != 'OpenEnv') ELSE badges END AS badges_q, + CASE WHEN kind = 'space' THEN (CASE WHEN {verified} THEN 'openenv' WHEN framework IS 'openenv' OR openenv THEN 'unverified-space' ELSE 'other-space' END) + ELSE COALESCE(framework, 'harbor') END AS fw, + CASE WHEN kind = 'space' AND (framework IS 'openenv' OR openenv) + THEN CASE WHEN manifest IS NOT NULL THEN 'Manifest present' ELSE 'Hub tag only' END END AS evidence_f, + CASE WHEN kind = 'space' THEN {stage} END AS runtime_f, + CASE WHEN kind = 'space' THEN CASE WHEN mcp THEN 'MCP tagged' ELSE 'No MCP tag' END END AS mcp_tag_f, + CASE WHEN kind = 'space' AND (framework IS 'openenv' OR openenv OR ({verified})) + THEN COALESCE(json_extract(check_json, '$.status'), 'Not checked') END AS health_f, + CASE WHEN json_extract(check_json, '$.status') = 'API checked' THEN json_extract(check_json, '$.mode') END AS mode_f, + CASE WHEN kind = 'space' AND (framework IS 'openenv' OR openenv OR ({verified})) THEN + CASE WHEN json_extract(check_json, '$.status') IN ('API checked', 'Checks failed') THEN + CASE WHEN json_extract(check_json, '$.tools') > 0 THEN 'Tools discovered' ELSE 'No tools discovered' END + ELSE 'Not checked' END END AS tools_f, + CASE WHEN kind = 'space' AND (framework IS 'openenv' OR openenv OR ({verified})) THEN + COALESCE(NULLIF(json_extract(check_json, '$.version.value'), 'Unknown'), openenv_version, 'Unknown') END AS version_f, + CASE WHEN kind = 'space' AND (framework IS 'openenv' OR openenv OR ({verified})) THEN + CASE WHEN json_extract(check_json, '$.version.value') IS NOT NULL AND json_extract(check_json, '$.version.value') != 'Unknown' + THEN json_extract(check_json, '$.version.source') WHEN openenv_version IS NOT NULL THEN 'Hub tag' ELSE 'Unknown' END END AS version_source_f + FROM b0 ) -""" +""".replace("{stage}", snapshot.stage_sql("stage")).replace("{verified}", + "json_extract(check_json, '$.status') = 'API checked' AND json_extract(check_json, '$.stage') = 'RUNNING'") ROLLOUTS = "COALESCE((SELECT value FROM json_each(:rolls) WHERE key = b.id), 0)" @@ -214,33 +248,51 @@ TYPE_VALUES = "json_each(CASE WHEN json_array_length(b.badges_q) = 0 THEN '[\"Ne FACET_FILTER = { "type": f"EXISTS (SELECT 1 FROM {TYPE_VALUES} j WHERE j.value IN (SELECT value FROM json_each({{p}})))", "size": "b.size_f IN (SELECT value FROM json_each({p}))", - "stage": "b.stage_f IN (SELECT value FROM json_each({p}))", - "mcp": "b.mcp_f IN (SELECT value FROM json_each({p}))", - "oe": "b.oe_f IN (SELECT value FROM json_each({p}))", + "stage": "b.runtime_f IN (SELECT value FROM json_each({p}))", + "health": "b.health_f IN (SELECT value FROM json_each({p}))", + "mode": "b.mode_f IN (SELECT value FROM json_each({p}))", + "tools": "b.tools_f IN (SELECT value FROM json_each({p}))", + "mcp": "b.mcp_tag_f IN (SELECT value FROM json_each({p}))", + "oe": "b.version_f IN (SELECT value FROM json_each({p}))", + "oe_source": "b.version_source_f IN (SELECT value FROM json_each({p}))", + "evidence": "b.evidence_f IN (SELECT value FROM json_each({p}))", "tags": "EXISTS (SELECT 1 FROM json_each(b.tags) j WHERE j.value IN (SELECT value FROM json_each({p})))", } FACET_COUNT = { # FROM, value expression, extra condition "type": (f"base b, {TYPE_VALUES} j", "j.value", "1"), "size": ("base b", "b.size_f", "b.size_f IS NOT NULL"), - "stage": ("base b", "b.stage_f", "b.stage_f IS NOT NULL"), - "mcp": ("base b", "b.mcp_f", "b.mcp_f IS NOT NULL"), - "oe": ("base b", "b.oe_f", "b.oe_f IS NOT NULL"), + "stage": ("base b", "b.runtime_f", "b.runtime_f IS NOT NULL"), + "health": ("base b", "b.health_f", "b.health_f IS NOT NULL"), + "mode": ("base b", "b.mode_f", "b.mode_f IS NOT NULL"), + "tools": ("base b", "b.tools_f", "b.tools_f IS NOT NULL"), + "mcp": ("base b", "b.mcp_tag_f", "b.mcp_tag_f IS NOT NULL"), + "oe": ("base b", "b.version_f", "b.version_f IS NOT NULL"), + "oe_source": ("base b", "b.version_source_f", "b.version_source_f IS NOT NULL"), + "evidence": ("base b", "b.evidence_f", "b.evidence_f IS NOT NULL"), "tags": ("base b, json_each(b.tags) j", "j.value", "1"), } -TRENDING = "((b.pinned * 1e15) + ((b.trending * 1e9 + b.likes * 1e4) + MIN(b.downloads, 9999)))" +TRENDING = "b.trending" SORT_SQL = {"trending": TRENDING, "downloads": "b.downloads", "likes": "b.likes", "tasks": "COALESCE(b.tasks, -1)", "rollouts": ROLLOUTS, "updated": "b.updated_ms", "new": "b.created_ms"} def sort_sql(q: Query) -> str: - if q.sort == "trending" and q.kind == "openenv": # FineEnvs' curated servers lead the OpenEnv Spaces - return f"((b.coll IS 'fineenvs') * 1e14 + {TRENDING})" return SORT_SQL[q.sort] +def order_sql(q: Query) -> str: + return "sk DESC, " + ("b.likes DESC, b.downloads DESC, " if q.sort == "trending" else "") + "b.ord ASC" + + +def order_key(q: Query, d: dict[str, Any]) -> tuple: + return (-d["_sk"],) + ((-(d.get("likes") or 0), -(d.get("downloads") or 0)) if q.sort == "trending" else ()) + (d["_ord"],) + + def where(q: Query, skip: str | None = None, ignore_coll: bool = False, ignore_kind: bool = False) -> tuple[str, dict[str, Any]]: parts: list[str] = [] params: dict[str, Any] = {} + if q.scope == "ready": + parts.append("(b.kind = 'dataset' OR json_extract(b.check_json, '$.browseable') = 1)") if not ignore_kind and q.kind != "all": parts.append("b.fw = :kind") params["kind"] = q.kind @@ -250,6 +302,9 @@ def where(q: Query, skip: str | None = None, ignore_coll: bool = False, ignore_k else: parts.append("COALESCE(b.coll, 'other') = :coll") params["coll"] = q.coll + if q.owner: + parts.append("substr(b.id, 1, instr(b.id, '/') - 1) = :owner COLLATE NOCASE") + params["owner"] = q.owner words = q.words if words: fts = envs_fts_query(words) @@ -274,6 +329,9 @@ def card(r: Any, rolls: dict[str, int]) -> dict[str, Any]: "downloads": r["downloads"], "likes": r["likes"], "trending": r["trending"], "updated": r["updated"], "created": r["created"], "stage": r["stage"], "mcp": bool(r["mcp"]), "openenv_version": r["openenv_version"], "manifest": r["manifest"], "hardware": r["hardware"], "badges": json.loads(r["badges_q"]), "tags": json.loads(r["tags"]), + "evidence": r["evidence_f"], "stage_label": r["runtime_f"], + "api_status": r["health_f"], "api_check": json.loads(r["check_json"]) if r["check_json"] else None, + "declared_version": r["version_f"], "version_source": r["version_source_f"], "api_mode": r["mode_f"], "tools_status": r["tools_f"], "pinned": bool(r["pinned"]), "mine": bool(r["mine"]), "indexed": json.loads(r["indexed"]) if r["indexed"] else None, "rollouts": rolls.get(r["id"], 0), "private": False, "_sk": r["sk"], "_ord": r["ord"]} @@ -289,7 +347,7 @@ def sql_search(conn, q: Query, ctx: Ctx, *, facets: bool = True, limit: int | No w, p = where(q) lim = q.size if limit is None else limit off = (q.page - 1) * q.size if offset is None else offset - rows = conn.execute(f"{B} SELECT b.*, {sort_sql(q)} AS sk FROM base b WHERE {w} ORDER BY sk DESC, b.ord ASC LIMIT :lim OFFSET :off", + rows = conn.execute(f"{B} SELECT b.*, {sort_sql(q)} AS sk FROM base b WHERE {w} ORDER BY {order_sql(q)} LIMIT :lim OFFSET :off", {**base, **p, "lim": lim, "off": off}).fetchall() total = conn.execute(f"{B} SELECT COUNT(*) FROM base b WHERE {w}", {**base, **p}).fetchone()[0] out: dict[str, Any] = {"rows": [card(r, ctx.rolls) for r in rows], "total": total} @@ -317,22 +375,46 @@ def sql_search(conn, q: Query, ctx: Ctx, *, facets: bool = True, limit: int | No return out -def sql_extra(conn, ctx: Ctx, trending: int) -> dict[str, Any]: - """The page's header numbers, and the trending row (every environment, pinned first).""" +def sql_extra(conn, ctx: Ctx, trending: int, q: Query | None = None) -> dict[str, Any]: + """Counts, task coverage and editorial picks all follow the current search.""" base, B = ctx.params(), ctx.base() - ds, sp, tasks = conn.execute(f"{B} SELECT SUM(b.kind = 'dataset'), SUM(b.kind = 'space'), SUM(COALESCE(b.tasks, 0)) FROM base b", - base).fetchone() - out: dict[str, Any] = {"stats": {"datasets": ds or 0, "spaces": sp or 0, "tasks": tasks or 0}} + w, p = where(q or Query()) + params = {**base, **p} + ds, sp = conn.execute(f"{B} SELECT SUM(b.kind = 'dataset'), SUM(b.kind = 'space') FROM base b WHERE {w}", params).fetchone() + # Count actual searchable records, including adapters (MiMo) with no Harbor + # summary. Health discovery must never imply that tasks were downloaded. + tasks = conn.execute(f"{B} SELECT COUNT(*) FROM tasks t JOIN base b ON b.key=t.env WHERE b.kind='dataset' AND {w}", params).fetchone()[0] + from . import space_tasks + visible = [r[0] for r in conn.execute(f"{B} SELECT b.id FROM base b WHERE b.kind='space' AND {w}", params)] + space_counts = space_tasks.census(ctx.checks, visible) + out: dict[str, Any] = {"stats": {"datasets": ds or 0, "spaces": sp or 0, "tasks": tasks, + "dataset_tasks": tasks, "space_tasks": space_counts["tasks"], + "space_task_coverage": space_counts}} + out["stats"]["openenv"] = dict(conn.execute(f"{B} SELECT b.evidence_f, COUNT(*) FROM base b WHERE b.evidence_f IS NOT NULL AND {w} GROUP BY 1", params)) + out["stats"]["openenv_running"] = conn.execute(f"{B} SELECT COUNT(*) FROM base b WHERE b.evidence_f IS NOT NULL AND b.stage = 'RUNNING' AND {w}", params).fetchone()[0] if trending: - rows = conn.execute(f"{B} SELECT b.*, {TRENDING} AS sk FROM base b ORDER BY sk DESC, b.ord ASC LIMIT :n", {**base, "n": trending}) - out["trending"] = [card(r, ctx.rolls) for r in rows] + tq = replace(q or Query(), sort="trending", page=1, size=trending) + out["trending"] = sql_search(conn, tq, ctx, facets=False)["rows"] + w, p = where(tq) + rows = conn.execute(f"{B} SELECT b.*, 0 AS sk FROM base b WHERE b.pinned AND {w} " + "ORDER BY (SELECT CAST(key AS INTEGER) FROM json_each(:pins) WHERE value = b.key) LIMIT 12", {**base, **p}) + out["featured"] = [card(r, ctx.rolls) for r in rows] + # FineEnvs is an explicit editorial section, independent of admin pins + # and interest rank. Owner/search filters still apply to it. + promoted = conn.execute(f"{B} SELECT b.*, {TRENDING} AS sk FROM base b WHERE {w} " + "AND substr(b.id, 1, instr(b.id, '/') - 1) = 'FineEnvs' COLLATE NOCASE " + "AND (b.kind = 'dataset' OR json_extract(b.check_json, '$.browseable') = 1) " + "ORDER BY (b.kind = 'space') DESC, b.pinned DESC, " + order_sql(tq) + " LIMIT 4", {**base, **p}) + out["promoted"] = [card(r, ctx.rolls) for r in promoted] return out # ── the same, in Python: the visitor's own datasets that aren't in the snapshot, and the tests' reference ──────── -def py_fw(d: dict[str, Any]) -> str: +def py_fw(d: dict[str, Any], check=None) -> str: if d.get("kind") == "space": - return "openenv" if d.get("framework") == "openenv" or d.get("openenv") else "other-space" + if space_checks.verified(check): + return "openenv" + return "unverified-space" if d.get("framework") == "openenv" or d.get("openenv") else "other-space" return d.get("framework") or "harbor" @@ -341,15 +423,19 @@ PY_FACETS = { "size": lambda d: [snapshot.size_bucket((d.get("indexed") or {}).get("tasks"))] if d.get("kind") == "dataset" and (d.get("framework") or "harbor") == "harbor" else [], "stage": lambda d: [snapshot.stage_value(d.get("stage"))] if d.get("kind") == "space" else [], - "mcp": lambda d: (["Has MCP tools" if d.get("mcp") else "No MCP tag"]) if d.get("kind") == "space" else [], - "oe": lambda d: [d["openenv_version"]] if d.get("kind") == "space" and d.get("openenv_version") else [], + "mcp": lambda d: (["MCP tagged" if d.get("mcp") else "No MCP tag"]) if d.get("kind") == "space" else [], + "health": lambda d: [d["api_status"]] if d.get("api_status") else [], + "mode": lambda d: [d["api_mode"]] if d.get("api_mode") else [], + "tools": lambda d: [d["tools_status"]] if d.get("tools_status") else [], + "oe": lambda d: [d["declared_version"]] if d.get("declared_version") else [], + "oe_source": lambda d: [d["version_source"]] if d.get("version_source") else [], + "evidence": lambda d: [d["evidence"]] if d.get("evidence") else [], "tags": lambda d: d.get("tags") or [], } def py_cards(rows: list[dict[str, Any]], ctx: Ctx, ord0: int = 0) -> list[dict[str, Any]]: - """Listing rows as the query sees them: collection, pins, the visitor's, rollouts, and FineEnvs' curated Spaces - as OpenEnv (catalog.environments' rule). Hidden ones left out.""" + """Apply visibility and editorial metadata without changing framework evidence or rank.""" hidden, pins, mine = set(ctx.hidden), set(ctx.pins), set(ctx.mine) out = [] for i, r in enumerate(rows): @@ -360,9 +446,26 @@ def py_cards(rows: list[dict[str, Any]], ctx: Ctx, ord0: int = 0) -> list[dict[s d = {**r, "key": key, "collection": coll, "pinned": key in pins, "mine": key in mine or bool(r.get("mine")), "rollouts": ctx.rolls.get(str(r["id"]), 0), "_ord": ord0 + i, "_blob": snapshot.blob_of(r), "badges": list(r.get("badges") or []), "tags": list(r.get("tags") or [])} - if d.get("kind") == "space" and coll == "fineenvs" and r.get("framework") != "openenv": - d.update(framework="openenv", openenv=True, badges=["OpenEnv"]) - d["fw"] = py_fw(d) + check = ctx.checks.get(d["id"], {}) if d.get("kind") == "space" else {} + declared = d.get("kind") == "space" and (d.get("framework") == "openenv" or d.get("openenv")) + d["fw"] = py_fw(d, check) + d["evidence"] = ("Manifest present" if d.get("manifest") else "Hub tag only") if declared else None + d["stage_label"] = snapshot.stage_value(d.get("stage")) if d.get("kind") == "space" else None + health = space_checks.status(check) + candidate = declared or d["fw"] == "openenv" + ver = check.get("version") or {} + known_ver = ver.get("value") and ver["value"] != "Unknown" + d.update(api_check={**check, "status": health} if check else None, api_status=health if candidate else None, + api_mode=check.get("mode") if health == space_checks.PASS else None, + tools_status=("Tools discovered" if check.get("tools", 0) > 0 else "No tools discovered") + if candidate and health in (space_checks.PASS, "Checks failed") else "Not checked" if candidate else None, + declared_version=(ver["value"] if known_ver else d.get("openenv_version") or "Unknown") if candidate else None, + version_source=(ver.get("source") if known_ver else "Hub tag" if d.get("openenv_version") else "Unknown") if candidate else None) + if d.get("kind") == "space": + d["openenv"] = d["fw"] == "openenv" + d["framework"] = "openenv" if d["openenv"] else "ors" if d.get("framework") == "ors" else "space" + if not d["openenv"]: + d["badges"] = [b for b in d["badges"] if b != "OpenEnv"] out.append(d) return out @@ -372,22 +475,22 @@ def _time(iso: Any) -> int: def py_key(q: Query): - tk = lambda d: ((d.get("trending") or 0) * 1e9 + (d.get("likes") or 0) * 1e4) + min(d.get("downloads") or 0, 9999) - keys = {"trending": lambda d: (1e15 if d["pinned"] else 0) + tk(d), "downloads": lambda d: d.get("downloads") or 0, + keys = {"trending": lambda d: d.get("trending") or 0, "downloads": lambda d: d.get("downloads") or 0, "likes": lambda d: d.get("likes") or 0, "tasks": lambda d: ((d.get("indexed") or {}).get("tasks") if (d.get("indexed") or {}).get("tasks") is not None else -1), "rollouts": lambda d: d["rollouts"], "updated": lambda d: _time(d.get("updated")), "new": lambda d: _time(d.get("created"))} - k = keys[q.sort] - if q.sort == "trending" and q.kind == "openenv": - return lambda d: (1e14 if d.get("collection") == "fineenvs" else 0) + k(d) - return k + return keys[q.sort] def py_passes(d: dict[str, Any], q: Query, skip: str | None = None, ignore_coll: bool = False, ignore_kind: bool = False) -> bool: """The page's own `passes()`, line for line.""" + if q.scope == "ready" and d.get("kind") == "space" and not space_checks.browseable(d.get("api_check")): + return False if not ignore_kind and q.kind != "all" and d["fw"] != q.kind: return False if not ignore_coll and q.coll and not (d["mine"] if q.coll == "mine" else (d.get("collection") or "other") == q.coll): return False + if q.owner and d["id"].split("/", 1)[0].lower() != q.owner.lower(): + return False if q.words and not all(w in d["_blob"] for w in q.words): return False for k, vals in q.sel.items(): @@ -401,9 +504,10 @@ def py_passes(d: dict[str, Any], q: Query, skip: str | None = None, ignore_coll: def py_search(cards: list[dict[str, Any]], q: Query, *, with_mine: bool = False) -> dict[str, Any]: """`cards` (from py_cards) filtered, sorted and counted as the page did it: every match (sorted) and raw facets.""" key = py_key(q) - matches = sorted((d for d in cards if py_passes(d, q)), key=lambda d: (-key(d), d["_ord"])) + matches = [d for d in cards if py_passes(d, q)] for d in matches: d["_sk"] = key(d) + matches.sort(key=lambda d: order_key(q, d)) fc: dict[str, dict[Any, int]] = {"kind": {}, "collection": {}} for d in cards: if py_passes(d, q, ignore_kind=True): @@ -428,7 +532,8 @@ def py_search(cards: list[dict[str, Any]], q: Query, *, with_mine: bool = False) def _public_card(d: dict[str, Any]) -> dict[str, Any]: keep = ("id", "key", "kind", "framework", "fw", "openenv", "collection", "heading", "brief", "downloads", "likes", "trending", "updated", "created", "stage", "mcp", "openenv_version", "manifest", "hardware", "badges", "tags", - "pinned", "mine", "indexed", "rollouts", "private") + "pinned", "mine", "indexed", "rollouts", "private", "evidence", "stage_label", "api_status", "api_check", + "declared_version", "version_source", "api_mode", "tools_status") out = {k: d.get(k) for k in keep} out["openenv"], out["mcp"], out["private"] = bool(out["openenv"]), bool(out["mcp"]), bool(out["private"]) ix = d.get("indexed") @@ -557,7 +662,7 @@ def search(request: Request): res = sql_search(conn, q, ctx, facets=facets, limit=off + q.size, offset=0) more = py_search(py_cards([{**r, "key": r.get("key") or r["id"], "kind": "dataset", "mine": True} for r in extras], ctx, ord0=10**9), q, with_mine=True) - merged = sorted(res["rows"] + more["matches"], key=lambda d: (-d["_sk"], d["_ord"])) + merged = sorted(res["rows"] + more["matches"], key=lambda d: order_key(q, d)) res["rows"], res["total"] = merged[off:off + q.size], res["total"] + more["total"] if facets: for k, counts in more["facets"].items(): @@ -565,15 +670,17 @@ def search(request: Request): res["facets"][k][v] = res["facets"][k].get(v, 0) + n else: res = sql_search(conn, q, ctx, facets=facets) - head = sql_extra(conn, ctx, trending) if trending or facets else {} + head = sql_extra(conn, ctx, trending, q) if trending or facets else {} if extras and head: - head["stats"]["datasets"] += len(extras) - head["stats"]["tasks"] += sum((r.get("indexed") or {}).get("tasks") or 0 for r in extras) + head["stats"]["datasets"] += more["total"] + private_tasks = sum((r.get("indexed") or {}).get("tasks") or 0 for r in more["matches"]) + for metric in ("tasks", "dataset_tasks"): + head["stats"][metric] += private_tasks if trending: - tq = Query(sort="trending") + tq = replace(q, sort="trending") more = py_search(py_cards([{**r, "key": r.get("key") or r["id"], "kind": "dataset", "mine": True} for r in extras], ctx, ord0=10**9), tq)["matches"] - head["trending"] = sorted(head["trending"] + more, key=lambda d: (-d["_sk"], d["_ord"]))[:trending] + head["trending"] = sorted(head["trending"] + more, key=lambda d: order_key(tq, d))[:trending] info = {"built_at": snap.built_at, "source": snap.source} except snapshot.SnapshotError as e: raise HTTPException(503, f"the catalog isn't ready: {e}") @@ -593,6 +700,11 @@ def search(request: Request): out["stats"] = head["stats"] if trending: out["trending"] = [_public_card(d) for d in head.get("trending", [])] + out["featured"] = [_public_card(d) for d in head.get("featured", [])] + out["promoted"] = [_public_card(d) for d in head.get("promoted", [])] + out["ranking"] = RANKING + out["api_checks"] = {"fresh_seconds": space_checks.FRESH, "description": "Running on the Hub and OpenEnv API checks passed within the last hour. " + "No episode or reward validation. Versions come from repository dependencies or Hub tags, not the installed runtime."} out["snapshot"] = info if key is not None: with _answers_lock: @@ -636,19 +748,18 @@ def search_tasks(request: Request, q: str = "", env: str = "", page: int = 1, si @router.get("/api/search/rank") -def search_rank(key: str, request: Request): - """Where one environment stands among its kind (Spaces, or datasets) by the page's trending order, pins aside: +def search_rank(key: str, request: Request, scope: str = "all"): + """Where one environment stands among its kind (Spaces, or datasets) by the page's trending order: {n, of}. One query, so a page can say "#599 of 6,818 Spaces" without loading the whole listing.""" ctx = context() - score = "(%s.trending * 1e9 + %s.likes * 1e4 + MIN(%s.downloads, 9999))" - sql = ctx.base() + f""" -SELECT (SELECT COUNT(*) FROM base b WHERE b.kind = t.kind AND ({score % ('b', 'b', 'b')} > {score % ('t', 't', 't')} - OR ({score % ('b', 'b', 'b')} = {score % ('t', 't', 't')} AND b.ord < t.ord))) + 1, - (SELECT COUNT(*) FROM base b WHERE b.kind = t.kind) -FROM base t WHERE t.key = :key""" + w, params = where(Query(scope="ready" if scope == "ready" else "all")) + sql = ctx.base() + f""", ranked AS ( +SELECT key, ROW_NUMBER() OVER (PARTITION BY kind ORDER BY trending DESC, likes DESC, downloads DESC, ord) AS n, + COUNT(*) OVER (PARTITION BY kind) AS total FROM base b WHERE {w}) +SELECT n, total FROM ranked WHERE key = :key""" try: with snapshot.use() as (_snap, conn): - row = conn.execute(sql, {**ctx.params(), "key": key}).fetchone() + row = conn.execute(sql, {**ctx.params(), **params, "key": key}).fetchone() except snapshot.SnapshotError as e: raise HTTPException(503, f"the catalog isn't ready: {e}") if not row: diff --git a/app/seo.py b/app/seo.py index 9bae9b93dd418a5a38852b03cbce387a491b63b6..d82812d35a36a6e778549d01eb979596cfab6f7d 100644 --- a/app/seo.py +++ b/app/seo.py @@ -133,6 +133,8 @@ def kind_of(row: dict[str, Any] | None, spec: str = "") -> str: if not row: return "RL environment" if row.get("kind") == "space": + if row.get("fw") == "unverified-space": + return "Unverified Space" return {"openenv": KIND["openenv"], "ors": "ORS Space"}.get(row.get("framework") or ("openenv" if row.get("openenv") else ""), KIND["space"]) return KIND.get(row.get("framework") or "", "RL dataset") @@ -283,7 +285,7 @@ def space(request: Request, spec: str) -> HTMLResponse: "sameAs": f"https://huggingface.co/spaces/{spec}", "applicationCategory": "DeveloperApplication", "operatingSystem": "Web", "offers": {"@type": "Offer", "price": "0", "priceCurrency": "USD"}, "author": {"@type": "Organization", "name": spec.split("/")[0], "url": f"https://huggingface.co/{spec.split('/')[0]}"}, - "keywords": ["reinforcement learning", "RL environment", "OpenEnv", *((r or {}).get("tags") or [])[:10]]}, cld] + "keywords": ["reinforcement learning", "RL environment", *(["OpenEnv"] if (r or {}).get("openenv") else []), *((r or {}).get("tags") or [])[:10]]}, cld] return page(request, title=f"{name} · {kind}", description=desc, path=path, body=body, jsonld=ld, image=f"/og{path}.png" if r else None, noindex=not r) @@ -291,7 +293,7 @@ def simple(request: Request, path: str) -> HTMLResponse: """The app's other pages: community (indexed), a rollout, your rollouts, a comparison (not indexed: they change, or are personal).""" if path.startswith("/community"): return page(request, title="Community rollouts", path="/community", - description="Public rollouts of RL environments on Hugging Face, by every model: a leaderboard by environment, the traces, and the tasks nobody has tried yet.", + description="Public, graded RL rollouts across Harbor, MiMo, NeMo Gym and other supported runners. Explore traces and model results grouped by environment and scoring metric.", body="

Community rollouts

Public rollouts from everyone, shown without who ran them.

") titles = {"/runs": "My rollouts"} return page(request, title=titles.get(path, "Rollout" if path.startswith("/run/") else "Compare rollouts"), path=path, diff --git a/app/snapshot.py b/app/snapshot.py index 9a0a8d393352c844bc60c4bfb0bbea5ac4ffe1e5..0bd5cbe9a668b9707c62e38cf36f818fda8bbdb7 100644 --- a/app/snapshot.py +++ b/app/snapshot.py @@ -331,8 +331,21 @@ def size_bucket(n: int | None) -> str: return "Not indexed yet" if n is None else "Under 100" if n < 100 else "100 to 1,000" if n < 1000 else "1,000 or more" +SPACE_STAGES = { + "RUNNING": "Running", "SLEEPING": "Sleeping", "PAUSED": "Paused", "STOPPED": "Paused", + "BUILDING": "Building", "APP_STARTING": "Starting", "RUNNING_APP_STARTING": "Starting", + "RUNNING_BUILDING": "Building", "BUILD_ERROR": "Error", "RUNTIME_ERROR": "Error", + "CONFIG_ERROR": "Error", "NO_APP_FILE": "Error", "DELETING": "Deleting", +} + + def stage_value(stage: Any) -> str: - return "Running" if stage == "RUNNING" else "Asleep or stopped" if stage else "Unknown" + return SPACE_STAGES.get(stage, "Unknown") + + +def stage_sql(column: str) -> str: + """Also classify old snapshots from their raw runtime, without trusting the old broad stage_f bucket.""" + return "CASE " + column + " " + " ".join(f"WHEN '{k}' THEN '{v}'" for k, v in SPACE_STAGES.items()) + " ELSE 'Unknown' END" def blob_of(r: dict[str, Any]) -> str: diff --git a/app/space_checks.py b/app/space_checks.py new file mode 100644 index 0000000000000000000000000000000000000000..aa35d4d90e594f91a817149a45499ce45d3cc354 --- /dev/null +++ b/app/space_checks.py @@ -0,0 +1,358 @@ +"""Recent, non-executing OpenEnv API evidence, separate from the immutable catalog. + +No wake, reset, step or tool invocation. Only public running Spaces are probed. +Per-Space atomic files work on both local storage and the mounted bucket. Search +reads a bounded cached inventory; it never waits for the network. +""" +from __future__ import annotations + +import json +import logging +import os +import re +import threading +import time +import tomllib +from concurrent.futures import ThreadPoolExecutor + +import httpx +from packaging.requirements import InvalidRequirement, Requirement +from packaging.version import InvalidVersion, Version + +from . import catalog, config + +FRESH = 3600 +REFRESH = 1800 +PASS = "API checked" +UNKNOWN = "Not checked" +_cache = (None, 0.0, {}) +_lock = threading.Lock() +_stop = threading.Event() +_worker = None +log = logging.getLogger("rlx") + + +def _path(spec): + return config.STORAGE_DIR / "openenv-checks" / f"{catalog._slug(catalog.check_spec(spec))}.json" + + +def save(rec): + global _cache + path = _path(rec["id"]) + with _lock: + try: + old = json.loads(path.read_bytes()) + if old.get("checked_at", 0) > rec["checked_at"]: + return + except (OSError, ValueError): + pass + catalog.atomic_write(path, json.dumps(rec, separators=(",", ":")).encode()) + _cache = (None, 0.0, {}) + + +def inventory(): + """Public evidence only; invalid or oversized files are ignored, never trusted.""" + global _cache + directory = config.STORAGE_DIR / "openenv-checks" + with _lock: + if _cache[0] == directory and time.time() - _cache[1] < 15: + return _cache[2] + records = {} + for path in sorted(directory.glob("*.json"))[:20000]: + try: + if path.stat().st_size > 32 * 1024: + continue + r = json.loads(path.read_bytes()) + spec = catalog.check_spec(r["id"]) + if r.get("schema") == 1 and isinstance(r.get("checked_at"), (float, int)): + records[spec] = r + except (OSError, ValueError, KeyError, TypeError): + continue + _cache = (directory, time.time(), records) + return records + + +def status(rec, now=None): + if not rec: + return UNKNOWN + age = (time.time() if now is None else now) - rec.get("checked_at", 0) + if age < 0 or age >= FRESH: + return "Check expired" + return rec.get("status") if rec.get("status") in (PASS, "Checks failed", "Not running", "Check unavailable") else UNKNOWN + + +def verified(rec): + """Only a fresh successful check of a running server establishes API support.""" + return bool(rec and rec.get("stage") == "RUNNING" and status(rec) == PASS) + + +def browseable(rec): + """A live, recognized environment API; this does not certify reward quality.""" + return verified(rec) or bool(rec and rec.get("stage") == "RUNNING" + and status(rec) == "Checks failed" + and rec.get("interface") in ("nemo-gym", "ors")) + + +def environment_interface(api): + """Other supported server contracts, kept distinct from OpenEnv evidence.""" + if not isinstance(api, dict) or not isinstance(api.get("info"), dict) or not api["info"].get("version"): + return None + paths = api.get("paths") + if not isinstance(paths, dict): + return None + for name, routes in (("nemo-gym", ("/seed_session", "/verify")), ("ors", ("/create_session", "/{env_name}/call"))): + if all(isinstance(paths.get(p), dict) and isinstance(paths[p].get("post"), dict) for p in routes): + return name + return None + + +def tag_version(tags): + for tag in tags or []: + match = re.fullmatch(r"openenv-(\d[^\s]{0,70})", str(tag), re.I) + if match: + try: + return {"value": str(Version(match[1])), "source": "Hub tag"} + except InvalidVersion: + pass + return {"value": "Unknown", "source": "Unknown"} + + +def requirement_version(text): + try: + req = Requirement(text) + except InvalidRequirement: + return None + if req.name.lower().replace("_", "-") not in ("openenv", "openenv-core"): + return None # current OpenEnv and the earlier openenv-core distribution + if req.marker: + return {"value": "Conditional", "source": "Dependency constraint", "requirement": str(req)[:240]} + if req.url: + ref = re.search(r"@([^/#?\s]+)(?:#.*)?$", req.url) + return {"value": "git:" + ref[1][:64] if ref else "Unpinned source", "source": "Source reference"} + specs = list(req.specifier) + if len(specs) == 1 and specs[0].operator in ("==", "===") and "*" not in specs[0].version: + try: + return {"value": str(Version(specs[0].version)), "source": "Dependency pin"} + except InvalidVersion: + pass + return {"value": str(req.specifier) or "Unpinned", "source": "Dependency constraint"} + + +def parse_version(text, path): + """Repository evidence, never an assertion about the installed runtime.""" + if path.endswith(".toml") or path.endswith("uv.lock"): + try: + doc = tomllib.loads(text) + except (ValueError, RecursionError): + return None + if path.endswith("uv.lock"): + pins = [] + for pkg in doc.get("package", []): + if not isinstance(pkg, dict) or pkg.get("name", "").replace("_", "-") not in ("openenv", "openenv-core"): + continue + source = pkg.get("source") or {} + if source.get("git"): + ref = source["git"].rsplit("#", 1) + pins.append({"value": "git:" + ref[-1][:64] if len(ref) == 2 else "Unpinned source", "source": "Lockfile"}) + elif pkg.get("version"): + pins.append(requirement_version(f"openenv-core=={pkg['version']}")) + pins = [p for p in pins if p] + if len({p["value"] for p in pins}) == 1: + return {**pins[0], "source": "Lockfile"} + return {"value": "Multiple locked versions", "source": "Lockfile"} if pins else None + sources = doc.get("tool", {}).get("uv", {}).get("sources", {}) + source = sources.get("openenv") or sources.get("openenv-core") or sources.get("openenv_core") + if isinstance(source, dict) and source.get("git"): + ref = source.get("rev") or source.get("tag") or source.get("branch") + return {"value": "git:" + str(ref)[:64] if ref else "Unpinned source", "source": "Source reference"} + if source: + return {"value": "Custom source", "source": "Source reference"} + deps = doc.get("project", {}).get("dependencies", []) + else: + deps = [line.split(" #", 1)[0].strip() for line in text.splitlines()] + for dep in deps: + if isinstance(dep, str) and (v := requirement_version(dep)): + return v + return None + + +def declared_version(spec, tags): + from . import space_files as files + fallback = tag_version(tags) + try: + info = files._info(spec) + listing, _, _ = files.listing(info) + paths = [f["path"] for f in listing] + manifest = files.manifest_path(paths, "Dockerfile" if "Dockerfile" in paths else None, spec) + root = manifest.rsplit("/", 1)[0] if manifest and "/" in manifest else "" + keys = files.key_files(paths, root, manifest) + sizes = {f["path"]: f.get("size", 0) for f in listing} + reqs = [f"{root}/requirements.txt" if root else "requirements.txt", "requirements.txt"] + for path in dict.fromkeys([keys.get("lock"), keys.get("pyproject"), *reqs]): + if not path or path not in sizes or sizes[path] > 400 * 1024: + continue + text = files._fetch_text(spec, info["sha"], path, sizes[path]) + if text and (v := parse_version(text, path)): + return {**v, "path": path, "revision": info["sha"]} + except Exception: # incomplete version evidence must not change API health + pass + return fallback + + +def assess(api, health, metadata, schema, tools): + api = api if isinstance(api, dict) else {} + paths = api.get("paths") if isinstance(api.get("paths"), dict) else {} + def method(path, verb): + return isinstance(paths.get(path), dict) and isinstance(paths[path].get(verb), dict) + checks = { + "OpenAPI": isinstance(api.get("info"), dict) and isinstance(api["info"].get("version"), str), + "Health": isinstance(health, dict) and health.get("status") == "healthy", + "Metadata": isinstance(metadata, dict) and all(isinstance(metadata.get(k), str) for k in ("name", "description")), + "Schemas": isinstance(schema, dict) and all(isinstance(schema.get(k), dict) for k in ("action", "observation", "state")), + } + simulation = any(p in paths for p in ("/reset", "/step", "/state")) + mode = "Simulation" if simulation else "MCP service" + checks["Control routes" if simulation else "MCP tools"] = ( + all(method(p, v) for p, v in (("/reset", "post"), ("/step", "post"), ("/state", "get"))) + if simulation else method("/mcp", "post") and bool(tools)) + return {"status": PASS if all(checks.values()) else "Checks failed", "mode": mode if all(checks.values()) else None, + "failed": [k for k, ok in checks.items() if not ok], "tools": len(tools or [])} + + +def check(spec): + from . import spaces_live as live + rec = {"schema": 1, "id": catalog.check_spec(spec), "checked_at": time.time(), "status": "Check unavailable"} + previous = inventory().get(spec, {}) + rec["version"] = previous.get("version") or {"value": "Unknown", "source": "Unknown"} + try: + hub = live.record(spec, fresh=True) + rec["stage"] = hub["stage"] + if hub["stage"] != "RUNNING": + rec["status"] = "Not running" + elif hub.get("host"): + urls = ("/openapi.json", "/health", "/metadata", "/schema") + def get(path): + try: + with live.SpaceClient(timeout=httpx.Timeout(6, connect=4), follow_redirects=False) as client: + response = client.get(live._url(hub, path)) + return response.json() if response.status_code == 200 and "json" in response.headers.get("content-type", "") else None + except (httpx.HTTPError, ValueError, live.SpaceError): + return None + with ThreadPoolExecutor(4) as pool: + api, health, meta, schema = list(pool.map(get, urls)) + # MCP discovery is separate from typed-action OpenEnv compatibility. + paths = api.get("paths", {}) if isinstance(api, dict) else {} + typed = isinstance(schema, dict) and all(isinstance(schema.get(k), dict) for k in ("action", "observation", "state")) + tools, _, _ = live._tools({**hub, "mcp_standard": not typed}) if isinstance(paths, dict) and "/mcp" in paths else (None, None, None) + rec.update(assess(api, health, meta, schema, tools)) + rec["interface"] = environment_interface(api) + from . import space_tasks + task_api = space_tasks.discover(hub, paths) + rec["task_catalog"] = space_tasks.summary(task_api) + if rec["status"] == PASS: + rec["version"] = declared_version(spec, hub.get("tags")) + elif rec["version"]["value"] == "Unknown": + rec["version"] = tag_version(hub.get("tags")) + except Exception as exc: + rec["error"] = type(exc).__name__ # no upstream response/secret in public inventory + save(rec) + return rec + + +def observe(spec, info): + """Opening/rechecking a Space also updates discovery, including negative results.""" + try: + old = inventory().get(spec, {}) + rec = {"schema": 1, "id": spec, "checked_at": info.get("checked") or time.time(), "stage": info.get("stage"), + "version": old.get("version") or {"value": "Unknown", "source": "Unknown"}} + if not info.get("running"): + rec["status"] = "Not running" + else: + rec.update(assess({"info": {"version": info.get("openapi_version")}, "paths": info.get("endpoint_methods")}, + info.get("health"), info.get("metadata"), info.get("schema"), info.get("mcp"))) + rec["interface"] = environment_interface({"info": {"version": info.get("openapi_version")}, "paths": info.get("endpoint_methods")}) + from . import space_tasks + rec["task_catalog"] = space_tasks.summary(info.get("task_api")) + info["openenv_verified"] = verified(rec) + save(rec) + except Exception: + log.warning("Could not save OpenEnv check for %s", spec) + + +def scan(): + # App workers and an operator's CLI refresh share one bounded probe queue. + # Use local cache storage for this lock, never a mounted object-store bucket. + try: + import fcntl + except ImportError: # Windows development: the app still has one worker thread + return _scan() + config.CACHE_DIR.mkdir(parents=True, exist_ok=True) + with (config.CACHE_DIR / "openenv-checks.lock").open("a") as lease: + try: + fcntl.flock(lease, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return [] + return _scan() + + +def _scan(): + from . import snapshot + old = inventory() + with snapshot.use() as (_, conn): + candidates = list(conn.execute("SELECT id, stage FROM envs WHERE kind = 'space' ORDER BY trending DESC, likes DESC")) + due = pending(candidates, old) + def visit(spec): + if not _stop.is_set(): + return check(spec) + with ThreadPoolExecutor(4) as pool: + return [r for r in pool.map(visit, due) if r] + + +def pending(candidates, old, now=None): + """Bound the queue and reserve capacity for inactive candidates too. + + Never probing sleeping hosts, but refreshing their Hub metadata lets a + newly started Space enter the catalog without someone opening its page. + """ + now = time.time() if now is None else now + active, inactive = [], [] + for spec, stage in candidates: + rec = old.get(spec, {}) + running = rec.get("stage", stage) == "RUNNING" + at = rec.get("checked_at", 0) + interval = REFRESH if running else 6 * 3600 + if not rec or now - at >= interval or (running and status(rec, now) == PASS and "task_catalog" not in rec): + (active if running else inactive).append((spec, at)) + # Once checked, older entries take precedence so a failing popular Space + # cannot starve the long tail. FineEnvs goes first within each generation. + key = lambda item: (int(item[1] // REFRESH), item[0].split("/", 1)[0].lower() != "fineenvs", item[1]) + active.sort(key=key) + inactive.sort(key=key) + return [s for s, _ in active[:96] + inactive[:32]] + + +def start(): + global _worker + if os.environ.get("RLX_TEST_TMP") or os.environ.get("RLX_SPACE_CHECKS", "1") != "1" or (_worker and _worker.is_alive()): + return + _stop.clear() + def watch(): + while not _stop.is_set(): + try: + scan() + except Exception: + log.exception("OpenEnv API inventory refresh failed") + _stop.wait(60) + _worker = threading.Thread(target=watch, daemon=True, name="openenv-checks") + _worker.start() + + +def stop(): + _stop.set() + + +if __name__ == "__main__": + from collections import Counter + from . import runtime + runtime.hub_timeouts() + print(dict(Counter(r["status"] for r in scan())), flush=True) diff --git a/app/space_tasks.py b/app/space_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..993f6aa23f660406336ca3837607f4499ac6e4d0 --- /dev/null +++ b/app/space_tasks.py @@ -0,0 +1,94 @@ +"""Bounded discovery of the public OpenEnv Task API, without starting episodes. + +These are server-reported catalog sizes, not downloaded task records or proof that +an episode works. Keep their own timestamp so a health check cannot freshen them. +""" +from __future__ import annotations + +import re +import time +from concurrent.futures import ThreadPoolExecutor + +MAX_COUNT = 1_000_000_000 +FRESH = 3600 + + +def count(value): + return value if type(value) is int and 0 <= value <= MAX_COUNT else None + + +def discover(rec, paths, envs=None): + from . import spaces_live as live + + if "/{env_name}/splits" not in paths: + return None + if envs is None: + envs = live._get_json(rec, "/list_environments") + if not isinstance(envs, list): + return None + names = list(dict.fromkeys(e for e in envs if isinstance(e, str) and re.fullmatch(r"[\w.-]{1,80}", e) and e not in (".", ".."))) + + def read(name): + raw = live._get_json(rec, f"/{name}/splits") + splits, seen = [], set() + for s in raw[:40] if isinstance(raw, list) else []: + if not isinstance(s, dict) or not isinstance(s.get("name"), str) or len(s["name"]) > 200 or s["name"] in seen: + continue + seen.add(s["name"]) + n = count(s.get("num_tasks")) + if n is None and "/{env_name}/num_tasks" in paths and len(splits) < 12: + try: + r = live._client.post(live._url(rec, f"/{name}/num_tasks"), json={"split": s["name"]}, timeout=6) + doc = live._json(r) if r.status_code == 200 else None + n = count(doc.get("num_tasks")) if isinstance(doc, dict) else None + except Exception: + pass + splits.append({"name": s["name"], "type": s.get("type"), "num_tasks": n, "default": s.get("default") is True}) + return {"env": name, "splits": splits, "complete": isinstance(raw, list) and len(raw) <= 40 and len(seen) == len(raw)} + + with ThreadPoolExecutor(4) as pool: + environments = list(pool.map(read, names[:16])) + if not environments: + return None + return {"env": environments[0]["env"], "splits": environments[0]["splits"], "environments": environments, + "complete": len(names) <= 16 and all(e["complete"] for e in environments), "checked_at": time.time()} + + +def summary(task_api): + if not isinstance(task_api, dict): + return None + envs = task_api.get("environments") or [task_api] + entries, unknown, seen = [], 0, set() + for env in envs[:16]: + for split in env.get("splits", [])[:40]: + key = (env.get("env"), split.get("name")) + if key in seen: + continue + seen.add(key) + n = count(split.get("num_tasks")) + if n is None: + unknown += 1 + else: + entries.append(n) + return {"tasks": sum(entries), "counted_splits": len(entries), "unknown_splits": unknown, + "complete": bool(task_api.get("complete", True)) and not unknown, + "checked_at": task_api.get("checked_at") or time.time()} + + +def census(records, visible, now=None): + now = time.time() if now is None else now + out = {"tasks": 0, "spaces": 0, "partial_spaces": 0, "oldest_check": None} + for spec in visible: + rec = records.get(spec, {}) + s = rec.get("task_catalog") or {} + at = s.get("checked_at", 0) + if rec.get("stage") != "RUNNING" or not isinstance(at, (int, float)) or not 0 <= now - at < FRESH or not s.get("counted_splits"): + continue + n = s.get("tasks") + if type(n) is not int or not 0 <= n <= MAX_COUNT * 640: + continue + out["tasks"] += n + out["spaces"] += 1 + out["partial_spaces"] += not s.get("complete") + out["oldest_check"] = min(out["oldest_check"] or at, at) + return out diff --git a/app/spaces_live.py b/app/spaces_live.py index 31a1bdf54b42ed360705ff7d4c5a694e805e0e1a..3e0c76d8eff33ddbb19d405cd266ee70660ce979 100644 --- a/app/spaces_live.py +++ b/app/spaces_live.py @@ -20,6 +20,8 @@ import re import secrets import threading import time +import zlib +from urllib.parse import urlencode from concurrent.futures import ThreadPoolExecutor from typing import Any @@ -38,9 +40,6 @@ STANDARD_ROUTES = {"/reset", "/step", "/state", "/metadata", "/health", "/health "/{env_name}/splits", "/{env_name}/tasks", "/{env_name}/num_tasks", "/{env_name}/task", "/{env_name}/task_range", "/web/metadata", "/web/reset", "/web/step", "/web/state", "/docs", "/redoc", "/openapi.json"} MAX_REPLY = 12 * 2**20 # bytes from a Space for one reply (observations can carry images) -_client = httpx.Client(timeout=httpx.Timeout(12, connect=6), follow_redirects=False, - headers={"User-Agent": "hf-rl-explorer (+https://huggingface.co/FineEnvs)"}, - limits=httpx.Limits(max_connections=64, max_keepalive_connections=16)) _lock = threading.Lock() @@ -52,6 +51,48 @@ class SpaceError(Exception): self.status = status +class SpaceClient(httpx.Client): + """Bound decompressed replies while receiving them, including chunked bodies. + + Space operators control their responses. Checking Response.content after a + normal get() is too late: httpx has already buffered the complete response. + Keep httpx's cookie jar and connection pooling, but never buffer without a cap. + """ + + def send(self, request, *, stream=False, **kwargs): + # Negotiate only a codec with a bounded decompression API. + request.headers["Accept-Encoding"] = "gzip, identity" + response = super().send(request, stream=True, **kwargs) + try: + body = bytearray() + codec = response.headers.get("content-encoding", "identity").lower() + decoded = response.is_stream_consumed + if not decoded and codec not in {"identity", "gzip"}: + raise SpaceError("the Space used an unsupported response encoding", 502) + decoder = zlib.decompressobj(16 + zlib.MAX_WBITS) if not decoded and codec == "gzip" else None + chunks = (response.content,) if decoded else response.iter_raw(chunk_size=64 * 1024) + for chunk in chunks: + if decoder: + chunk = decoder.decompress(chunk, MAX_REPLY - len(body) + 1) + if len(body) + len(chunk) > MAX_REPLY: + raise SpaceError("the Space's reply is too large to show", 502) + body.extend(chunk) + if decoder and (not decoder.eof or decoder.unused_data): + raise SpaceError("the Space returned an invalid compressed response", 502) + headers = [(k, v) for k, v in response.headers.multi_items() + if k.lower() not in ("content-encoding", "content-length", "transfer-encoding")] + return httpx.Response(response.status_code, headers=headers, content=bytes(body), request=request) + except zlib.error: + raise SpaceError("the Space returned an invalid compressed response", 502) from None + finally: + response.close() + + +_client = SpaceClient(timeout=httpx.Timeout(12, connect=6), follow_redirects=False, + headers={"User-Agent": "hf-rl-explorer (+https://huggingface.co/FineEnvs)"}, + limits=httpx.Limits(max_connections=64, max_keepalive_connections=16)) + + # ── the Space's record ─────────────────────────────────────────────────────── def record(spec: str, fresh: bool = False) -> dict[str, Any]: """Host, stage and declared UI path of a public Space (cached a few seconds: the page polls it while it wakes).""" @@ -99,7 +140,7 @@ def _get_here(rec: dict, path: str, hops: int = 3) -> httpx.Response | None: if r.status_code not in (301, 302, 303, 307, 308): return r nxt = r.url.join(r.headers.get("location", "")) - if nxt.scheme != "https" or nxt.host != httpx.URL(rec["host"]).host: + if nxt.scheme != "https" or nxt.host != httpx.URL(rec["host"]).host or nxt.port not in (None, 443) or nxt.userinfo: return None url = str(nxt) return None @@ -133,6 +174,13 @@ def _get_json(rec: dict, path: str) -> Any: def mcp_call(rec: dict, method: str, params: dict | None = None, timeout: float = 30, raw: bool = False) -> Any: """One JSON-RPC call to the Space's /mcp. Returns `result`; raises SpaceError with the server's message on error. `raw`: the whole JSON-RPC reply, an error reply included (HTTP failures still raise).""" + if rec.get("mcp_standard"): + with SpaceClient(timeout=httpx.Timeout(timeout, connect=6), follow_redirects=False) as client: + remote = RemoteMCP(rec, client) + try: + return remote.call(method, params or {}, timeout, raw=raw) + finally: + remote.close() try: r = _client.post(_url(rec, "/mcp"), timeout=httpx.Timeout(timeout, connect=6), headers={"Accept": "application/json, text/event-stream", "Content-Type": "application/json"}, @@ -157,6 +205,100 @@ def mcp_call(rec: dict, method: str, params: dict | None = None, timeout: float return doc.get("result") +class RemoteMCP: + """A Streamable HTTP MCP session, isolated along with the visitor's cookies. + + No credentials from the explorer are forwarded. Expiry never replays an action: + callers must explicitly open a new episode after a 410 response. + """ + def __init__(self, rec: dict, client: httpx.Client): + self.url = _url(rec, "/mcp") + self.client = client + self.session = None + self.version = None + self.initialized = False + self.expired = False + + def headers(self): + headers = {"Accept": "application/json, text/event-stream", "Content-Type": "application/json"} + if self.session: + headers["Mcp-Session-Id"] = self.session + if self.version: + headers["MCP-Protocol-Version"] = self.version + return headers + + def request(self, method, params, timeout, *, notification=False): + rid = secrets.randbelow(10**9) + message = {"jsonrpc": "2.0", "method": method, "params": params} + if not notification: + message["id"] = rid + try: + r = self.client.post(self.url, headers=self.headers(), json=message, timeout=httpx.Timeout(timeout, connect=6)) + except httpx.TimeoutException: + raise SpaceError("the MCP server timed out; the action was not retried", 504) from None + except httpx.HTTPError as e: + raise SpaceError(f"couldn't reach the MCP server: {type(e).__name__}", 502) from None + if self.session and r.status_code == 404: + self.expired = True + raise SpaceError("the MCP session has expired: start a new session", 410) + if notification and r.status_code in (200, 202, 204): + return {} + if r.status_code != 200: + raise SpaceError(f"the Space's /mcp answered HTTP {r.status_code}", 502) + try: + if "text/event-stream" in r.headers.get("content-type", ""): + # Notifications can precede the response; SSE data may span lines. + docs = [json.loads("\n".join(line[5:].lstrip() for line in event.splitlines() if line.startswith("data:"))) + for event in r.text.replace("\r\n", "\n").split("\n\n") if any(line.startswith("data:") for line in event.splitlines())] + doc = next((d for d in docs if isinstance(d, dict) and d.get("id") == rid), None) + else: + doc = r.json() + except ValueError: + raise SpaceError("the MCP server returned invalid JSON", 502) from None + if not isinstance(doc, dict) or doc.get("jsonrpc") != "2.0" or doc.get("id") != rid: + raise SpaceError("the MCP server returned no matching JSON-RPC response", 502) + if method == "initialize" and not doc.get("error"): + sid = r.headers.get("mcp-session-id") + if sid and (len(sid) > 2048 or not all(0x21 <= ord(c) <= 0x7e for c in sid)): + raise SpaceError("the MCP server returned an invalid session ID", 502) + self.session = sid + return doc + + def call(self, method, params, timeout, *, raw=False): + if self.expired: + raise SpaceError("the MCP session has expired: start a new session", 410) + if not self.initialized: + init = self.request("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "hf-rl-explorer", "version": "0.1.0"}}, timeout) + error = init.get("error") or {} + if not isinstance(error, dict): + raise SpaceError("the MCP server returned an invalid initialization error", 502) + if error.get("code") != -32601: # compatibility with simple JSON-RPC tool servers + result = init.get("result") or {} + if not isinstance(result, dict): + raise SpaceError("the MCP server returned an invalid initialization result", 502) + self.version = result.get("protocolVersion") + if error or self.version not in {"2025-03-26", "2025-06-18", "2025-11-25"}: + raise SpaceError("the MCP server could not negotiate a supported protocol", 502) + self.request("notifications/initialized", {}, timeout, notification=True) + self.initialized = True + doc = self.request(method, params, timeout) + if raw: + return doc + if doc.get("error"): + error = doc["error"] + raise SpaceError(str(error.get("message", "MCP error") if isinstance(error, dict) else error)[:500], 422) + return doc.get("result") + + def close(self): + if self.session and not self.expired: + try: + self.client.delete(self.url, headers=self.headers(), timeout=3) + except Exception: # best-effort cleanup of an untrusted remote server + pass + self.expired = True + + def _tools(rec: dict) -> tuple[list[dict] | None, str | None, dict]: """Its MCP tools (or why there are none), and how /mcp answered: a JSON-RPC 2.0 reply, as `openenv validate` asks of it, or the failure.""" @@ -182,8 +324,6 @@ def _api_routes(api: Any, openenv: bool) -> list[dict[str, Any]]: own plumbing (geoguesser's /geoguesser/task/{i} returns the answer's coordinates). GET and POST only.""" if openenv or not isinstance(api, dict) or not isinstance(api.get("paths"), dict): return [] - comps = ((api.get("components") or {}).get("schemas") or {}) if isinstance(api.get("components"), dict) else {} - def resolve(x: Any, depth: int = 0) -> Any: if not isinstance(x, (dict, list)): return x @@ -191,7 +331,12 @@ def _api_routes(api: Any, openenv: bool) -> list[dict[str, Any]]: return {} if isinstance(x, dict) else [] if isinstance(x, dict): if "$ref" in x and isinstance(x["$ref"], str): - return resolve(comps.get(x["$ref"].split("/")[-1], {}), depth + 1) + target = api + if not x["$ref"].startswith("#/"): + return {} + for key in x["$ref"][2:].split("/"): + target = target.get(key.replace("~1", "/").replace("~0", "~"), {}) if isinstance(target, dict) else {} + return resolve(target, depth + 1) return {k: resolve(v, depth + 1) for k, v in x.items() if k != "$defs"} if isinstance(x, list): return [resolve(v, depth + 1) for v in x[:50]] @@ -203,7 +348,9 @@ def _api_routes(api: Any, openenv: bool) -> list[dict[str, Any]]: if (not isinstance(ops, dict) or not isinstance(path, str) or not re.match(r"^/(?!/)[\w\-.~{}/]{0,199}$", path) or ".." in path): continue - if path in STANDARD_ROUTES or path.startswith(("/web", "/login", "/logout", "/docs", "/redoc", "/static", "/gradio", "/assets")): + # A generic Gymnasium server's reset/step are ordinary HTTP routes, not + # evidence that it implements OpenEnv's WebSocket envelope protocol. + if path in STANDARD_ROUTES - {"/reset", "/step", "/state"} or path.startswith(("/web", "/login", "/logout", "/docs", "/redoc", "/static", "/gradio", "/assets")): continue if openenv and path in ("/healthz",): continue @@ -213,15 +360,24 @@ def _api_routes(api: Any, openenv: bool) -> list[dict[str, Any]]: continue body = (((op.get("requestBody") or {}).get("content") or {}).get("application/json") or {}).get("schema") schema = resolve(body) if body else None + parameters = {} + for p in [*(ops.get("parameters") or []), *(op.get("parameters") or [])]: + p = resolve(p) + if isinstance(p, dict) and p.get("in") == "query" and isinstance(p.get("name"), str): + parameters[p["name"]] = p + query = {"type": "object", "properties": {k: {**(p.get("schema") or {}), "description": p.get("description") or "Query parameter"} + for k, p in parameters.items()}, + "required": [k for k, p in parameters.items() if p.get("required")], "additionalProperties": False} out.append({"method": method.upper(), "path": path, "summary": str(op.get("summary") or op.get("description") or "")[:200], "params": re.findall(r"\{([A-Za-z_][\w-]{0,40})\}", path), - "schema": schema if isinstance(schema, dict) and schema.get("type") == "object" else None}) + "schema": schema if isinstance(schema, dict) else None, + "body_required": bool((op.get("requestBody") or {}).get("required")), "query_schema": query}) if len(out) >= 40: break return out -def route_url(info: dict, method: str, path: str, params: dict[str, Any]) -> str: +def route_url(info: dict, method: str, path: str, params: dict[str, Any], query: dict[str, Any] | None = None) -> str: """A route the server published, its path parameters filled in safely; anything else is refused.""" route = next((r for r in info.get("api") or [] if r["method"] == method and r["path"] == path), None) if route is None: @@ -229,9 +385,29 @@ def route_url(info: dict, method: str, path: str, params: dict[str, Any]) -> str filled = path for name in route["params"]: v = str((params or {}).get(name, "")) - if not re.match(r"^[\w.\-~]{1,200}$", v): + if not re.fullmatch(r"[\w.\-~]{1,200}", v) or v in (".", ".."): raise SpaceError(f"{name}: letters, digits, . - _ ~ only", 400) filled = filled.replace("{" + name + "}", v) + query = query or {} + schema = route.get("query_schema") or {} + if set(query) - set(schema.get("properties") or {}): + raise SpaceError("only this route's published query parameters are allowed", 400) + for key in schema.get("required") or []: + if key not in query: + raise SpaceError(f"query parameter {key} is required", 400) + pairs = [] + for key, value in query.items(): + for v in value if isinstance(value, list) else [value]: + if v is None: + continue + if not isinstance(v, (str, int, float, bool)): + raise SpaceError(f"query parameter {key} must be a scalar or array of scalars", 400) + pairs.append((key, str(v).lower() if isinstance(v, bool) else str(v))) + encoded = urlencode(pairs) + if len(encoded) > 8000: + raise SpaceError("query parameters are too large", 400) + if encoded: + filled += "?" + encoded return filled @@ -239,8 +415,12 @@ def probe(spec: str, fresh: bool = False) -> dict[str, Any]: """Everything a running server says about itself, in one record. Asleep or broken: just the stage.""" rec = record(spec, fresh=fresh) if rec["stage"] != UP or not rec["host"]: - return {"running": False, "stage": rec["stage"], "host": rec["host"], "base_path": rec["base_path"], + from .envs.capabilities import space_support + info = {"running": False, "stage": rec["stage"], "host": rec["host"], "base_path": rec["base_path"], "last_seen": last_seen(rec["id"])} + from . import space_checks + space_checks.observe(spec, info) + return {**info, "support": space_support(info)} def fetch(): with ThreadPoolExecutor(6) as pool: @@ -253,23 +433,11 @@ def probe(spec: str, fresh: bool = False) -> dict[str, Any]: api, meta, schema, health, envs, ui = (f.result() for f in (f_api, f_meta, f_schema, f_health, f_envs, f_ui)) routes = sorted((api or {}).get("paths", {}).keys()) if isinstance(api, dict) else [] has = set(routes) - openenv = isinstance(schema, dict) and ("action" in schema or "observation" in schema) - tools, tools_error, rpc = _tools(rec) if ("/mcp" in has or openenv) else (None, None, None) + openenv = isinstance(schema, dict) and all(isinstance(schema.get(k), dict) for k in ("action", "observation", "state")) + tools, tools_error, rpc = _tools({**rec, "mcp_standard": not openenv}) if ("/mcp" in has or openenv) else (None, None, None) env_names = [e for e in envs if isinstance(e, str) and re.match(r"^[\w.-]{1,80}$", e)] if isinstance(envs, list) else [] - task_api = bool(env_names) and "/{env_name}/splits" in has - splits = [] - if task_api: - raw = _get_json(rec, f"/{env_names[0]}/splits") - for s in (raw or [])[:40] if isinstance(raw, list) else []: - if isinstance(s, dict) and isinstance(s.get("name"), str): - splits.append({"name": s["name"], "type": s.get("type"), "num_tasks": s.get("num_tasks"), "default": s.get("default")}) - missing = [s for s in splits if not isinstance(s["num_tasks"], int)][:12] - for s in missing: - try: - r = _client.post(_url(rec, f"/{env_names[0]}/num_tasks"), json={"split": s["name"]}, timeout=8) - s["num_tasks"] = int(_json(r).get("num_tasks")) if r.status_code == 200 else None - except Exception: # noqa: BLE001 - s["num_tasks"] = None + from . import space_tasks + task_api = space_tasks.discover(rec, has, env_names) names = {t["name"] for t in tools or []} harbor = {"run_rollout", "capabilities"} <= names or "harbor_env" in env_names caps = _harbor_caps(rec) if "capabilities" in names and harbor else None @@ -277,30 +445,37 @@ def probe(spec: str, fresh: bool = False) -> dict[str, Any]: version = (api.get("info") or {}).get("version") if isinstance(api, dict) and isinstance(api.get("info"), dict) else None info = { "running": True, "stage": rec["stage"], "host": rec["host"], "base_path": rec["base_path"], - "ui": ui, "openenv": openenv or "/reset" in has, + "ui": ui, "openenv": openenv, "metadata": meta if isinstance(meta, dict) else None, "health": health if isinstance(health, dict) else None, "schema": schema if isinstance(schema, dict) else None, - "step_api": "/reset" in has and "/step" in has, "docs": "/openapi.json" in has or api is not None, + "step_api": openenv and "/reset" in has and "/step" in has, "docs": "/openapi.json" in has or api is not None, "mcp": tools, "mcp_error": None if tools else tools_error, - "task_api": {"env": env_names[0], "splits": splits} if task_api else None, + "task_api": task_api, "harbor": harbor, "harbor_caps": caps, "rewards": bool(isinstance(obs, dict) and "reward" in (obs.get("properties") or {})), "graders": sorted(n for n in names if re.search(r"(grade|score|submit|verify|reward|judge)", n, re.I)), "routes": [r for r in routes if r not in STANDARD_ROUTES][:40], - "api": _api_routes(api, openenv or "/reset" in has), + "api": _api_routes(api, openenv), # which framework the server speaks, from what it serves - "framework": "harbor" if harbor else "openenv" if (openenv or "/reset" in has) else + "framework": "harbor" if harbor else "openenv" if openenv else "nemo-gym" if {"/seed_session", "/verify"} <= has else - "ors" if {"/create_session", "/{env_name}/call"} <= has else "mcp" if tools else "api" if api else None, + "ors" if {"/create_session", "/{env_name}/call"} <= has else + "gymnasium" if {"/reset", "/step"} <= has else "mcp" if tools else "api" if api else None, "openapi_version": version if isinstance(version, str) else None, + "endpoint_methods": {p: {m: {} for m in methods if m in ("get", "post")} for p, methods in (api or {}).get("paths", {}).items() + if p in ("/reset", "/step", "/state", "/mcp") and isinstance(methods, dict)} if isinstance(api, dict) else {}, # OpenEnv's two modes: simulation serves /reset, /step and /state; production serves only MCP and /ws - "mode": ("simulation" if "/reset" in has else "production") if routes else None, + "mode": ("simulation" if "/reset" in has else "production") if openenv else None, "conformance": conformance({"openapi": api, "health": health, "metadata": meta, "schema": schema, "mcp": rpc, "mcp_tried": "/mcp" in has or openenv, "paths": routes}), "checked": time.time(), } if api is not None or isinstance(schema, dict) or isinstance(meta, dict) or tools: # it answered: keep it remember(rec["id"], info) + from .envs.capabilities import space_support + info["support"] = space_support(info) + from . import space_checks + space_checks.observe(spec, info) return info if fresh: @@ -518,17 +693,20 @@ def _withhold(v: Any, depth: int = 0) -> tuple[Any, list[str]]: return v, gone -def _task_api(spec: str) -> tuple[dict, str]: +def _task_api(spec: str, env: str = "") -> tuple[dict, str]: info = probe(spec) if not info.get("running"): raise SpaceError("this Space isn't running: wake it first", 409) if not info.get("task_api"): raise SpaceError("this server has no Task API", 404) - return record(spec), info["task_api"]["env"] + api = info["task_api"] + if env and env not in [e["env"] for e in api.get("environments", [api])]: + raise SpaceError("this environment is not in the Space's Task API", 400) + return record(spec), env or api["env"] -def tasks(spec: str, split: str, start: int, stop: int) -> dict[str, Any]: - rec, env = _task_api(spec) +def tasks(spec: str, split: str, start: int, stop: int, env: str = "") -> dict[str, Any]: + rec, env = _task_api(spec, env) start, stop = max(0, int(start)), max(0, int(stop)) stop = min(stop, start + 50) try: @@ -542,8 +720,8 @@ def tasks(spec: str, split: str, start: int, stop: int) -> dict[str, Any]: return {"tasks": rows, "start": start, "stop": stop, "withheld": sorted(set(gone))} -def task(spec: str, split: str, index: int) -> dict[str, Any]: - rec, env = _task_api(spec) +def task(spec: str, split: str, index: int, env: str = "") -> dict[str, Any]: + rec, env = _task_api(spec, env) try: r = _client.post(_url(rec, f"/{env}/task"), json={"split": split, "index": int(index)}, timeout=20) except httpx.HTTPError as e: @@ -570,12 +748,18 @@ class Session: self.created = self.used = time.time() self.ws = None self.ws_failed = False + self.broken = False # a dropped episode may only be recovered by an explicit reset self.http: httpx.Client | None = None # the session's own cookie jar for the server's routes (NeMo Gym keeps state in a cookie) + self.remote_mcp: RemoteMCP | None = None + self.standard_mcp = False self.lock = threading.Lock() self.steps = 0 self.rpc = 0 def close(self) -> None: + if self.remote_mcp is not None: + self.remote_mcp.close() + self.remote_mcp = None if self.http is not None: try: self.http.close() @@ -625,6 +809,8 @@ def start(spec: str, owner: str) -> dict[str, Any]: if len(_sessions) >= TOTAL: raise SpaceError("too many live sessions on this explorer right now: try again in a few minutes", 429) s = Session(catalog.check_spec(spec), owner) + s.ws_failed = not bool(info.get("openenv")) + s.standard_mcp = not bool(info.get("openenv")) _sessions[s.id] = s return {"session": s.id} @@ -660,6 +846,10 @@ def _open(s: Session) -> None: def _ws_send(s: Session, msg: dict, timeout: float) -> dict: + if s.broken and msg.get("type") != "reset": + raise SpaceError("the episode was disconnected: reset to start a new episode", 410) + if msg.get("type") == "reset": + s.broken = False if s.ws is None: _open(s) try: @@ -667,9 +857,11 @@ def _ws_send(s: Session, msg: dict, timeout: float) -> dict: reply = json.loads(s.ws.recv(timeout=timeout)) except TimeoutError: s.close() + s.broken = True raise SpaceError(f"the Space didn't answer within {int(timeout)} s; the session was closed", 504) from None except Exception as e: # noqa: BLE001 - the socket dropped: the episode is gone s.close() + s.broken = True raise SpaceError(f"the session dropped ({type(e).__name__}): start a new episode", 502) from None if not isinstance(reply, dict): raise SpaceError("the Space answered something unexpected", 502) @@ -680,7 +872,7 @@ def _ws_send(s: Session, msg: dict, timeout: float) -> dict: def _rpc(s: Session, method: str, params: dict, timeout: float) -> Any: - """An MCP call inside the session's episode; over plain HTTP (stateless) when the server has no socket for it.""" + """Use OpenEnv's episode socket, standard HTTP MCP, or legacy OpenEnv HTTP fallback.""" if not s.ws_failed: try: s.rpc += 1 @@ -696,6 +888,12 @@ def _rpc(s: Session, method: str, params: dict, timeout: float) -> Any: err = data["error"] if isinstance(data["error"], dict) else {"message": str(data["error"])} raise SpaceError(str(err.get("message") or "error")[:600], 422) return data.get("result") if isinstance(data, dict) else data + if s.standard_mcp: + if s.http is None: + s.http = SpaceClient(timeout=httpx.Timeout(120, connect=8), follow_redirects=False) + if s.remote_mcp is None: + s.remote_mcp = RemoteMCP(record(s.spec), s.http) + return s.remote_mcp.call(method, params, timeout) return mcp_call(record(s.spec), method, params, timeout=timeout) @@ -705,14 +903,15 @@ def _http(s: Session, data: dict) -> dict[str, Any]: if method not in ("GET", "POST"): raise SpaceError("GET or POST only", 400) info = probe(s.spec) - path = route_url(info, method, str(data.get("path") or ""), data.get("params") if isinstance(data.get("params"), dict) else {}) + path = route_url(info, method, str(data.get("path") or ""), data.get("params") if isinstance(data.get("params"), dict) else {}, + data.get("query") if isinstance(data.get("query"), dict) else {}) rec = record(s.spec) if s.http is None: - s.http = httpx.Client(base_url=rec["host"], timeout=httpx.Timeout(120, connect=8), follow_redirects=False, + s.http = SpaceClient(base_url=rec["host"], timeout=httpx.Timeout(120, connect=8), follow_redirects=False, headers={"User-Agent": "hf-rl-explorer (+https://huggingface.co/FineEnvs)"}) body = data.get("body") try: - r = s.http.request(method, path, json=body if method == "POST" else None) + r = s.http.request(method, _url(rec, path), json=body if method == "POST" else None) except httpx.TimeoutException: raise SpaceError("the Space didn't answer within 120 s", 504) from None except httpx.HTTPError as e: @@ -773,7 +972,8 @@ def act(sid: str, owner: str, op: str, data: dict | None = None) -> dict[str, An s.steps += 1 else: raise SpaceError(f"unknown operation {op!r}", 400) - return {"result": out, "ms": int((time.time() - t0) * 1000), "steps": s.steps, "stateful": not s.ws_failed} + return {"result": out, "ms": int((time.time() - t0) * 1000), "steps": s.steps, + "stateful": op == "http" or s.standard_mcp or not s.ws_failed} def live_sessions() -> int: diff --git a/app/vendor/nemo_gym/LICENSE b/app/vendor/nemo_gym/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/app/vendor/nemo_gym/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/app/vendor/nemo_gym/NOTICE b/app/vendor/nemo_gym/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..9ac74c602c31d5c7acb9ee893a6630d408214452 --- /dev/null +++ b/app/vendor/nemo_gym/NOTICE @@ -0,0 +1,5 @@ +NeMo Gym verification_utils.py from NVIDIA-NeMo/Gym +Commit: 3ef478df1ee163134d32a3f291f0a9e5981d0e52 +Path: resources_servers/single_step_tool_use_with_argument_comparison/common/verification_utils.py +SHA256: 41d82a49130adbf69fa9fca86c5f3eed28992ea6909e9dc05131b2830c0a1f89 +Unmodified. Copyright NVIDIA CORPORATION & AFFILIATES. Apache-2.0. diff --git a/app/vendor/nemo_gym/verification_utils.py b/app/vendor/nemo_gym/verification_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c5275eab6a702e9e60c2f0dccbbdb56c026908df --- /dev/null +++ b/app/vendor/nemo_gym/verification_utils.py @@ -0,0 +1,398 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from collections import Counter +from enum import StrEnum +from json import JSONDecodeError +from typing import Annotated, Any, Literal, Optional, TypeAlias, Union + +from pydantic import BaseModel, Field + + +class MessageAction(BaseModel): + type: Literal["message"] + content: str + + +class FunctionCallAction(BaseModel): + type: Literal["function_call"] + name: str + arguments: str + + +class FunctionCallBatchAction(BaseModel): + type: Literal["function_call_batch"] + calls: list[FunctionCallAction] = Field(min_length=1) + + +# Actions are canonical on both sides of a comparison: dataset rows deserialize into them, and model +# responses are normalized into them by `response_utils.extract_action`. The alias keeps the name +# that dataset validation tooling imports. +ExpectedAction: TypeAlias = Annotated[ + Union[MessageAction, FunctionCallAction, FunctionCallBatchAction], + Field(discriminator="type"), +] + + +def get_tool_calls(action: ExpectedAction) -> list[FunctionCallAction]: + """Flatten an action into the tool calls it represents, so single and parallel calls share a path.""" + if isinstance(action, FunctionCallBatchAction): + return list(action.calls) + + if isinstance(action, FunctionCallAction): + return [action] + + return [] + + +class StepRewardCategory(StrEnum): + NO_ACTION_FOUND = "No tool call or chat message was found in the response" + NO_EXPECTED_TOOL_CALL = "No tool call was found when one was expected" + EXPECTED_CHAT_MESSAGE_FOUND = "A chat message was found as expected" + NO_EXPECTED_CHAT_MESSAGE = "A tool call was executed when a chat message was expected" + UNEXPECTED_TOOL = "The tool in a tool call is not the expected tool" + ARGUMENTS_DECODE_ERROR = "An error occurred when decoding the arguments string in a tool call as a JSON object" + ARGUMENT_VALUE_TYPE_DIFFERENT = "The type of an argument value in a tool call is different than the expected type" + ARGUMENT_OBJECT_KEYS_DIFFERENT = ( + "The keys in an object in an argument value in a tool call are different than the keys in the expected object" + ) + ARGUMENT_LIST_LENGTH_DIFFERENT = ( + "A list in an argument value in a tool call has a different length than the expected list" + ) + ARGUMENT_VALUE_DIFFERENT = "An argument value in a tool call is different than the expected value" + EXPECTED_TOOL_CALL = "A tool call that matches the expected tool call was found" + FUNCTION_CALL_BATCH_LENGTH_DIFFERENT = "The number of tool calls in a batch is different than expected" + EXPECTED_TOOL_CALL_BATCH = "A tool-call batch that matches the expected tool calls was found" + + +class ParallelToolCallRewardMode(StrEnum): + """How an admissible parallel tool-call response converts its matched-call count into a reward.""" + + BINARY_STRICT = "binary_strict" + FRACTIONAL = "fractional" + F1 = "f1" + + +class ActionComparisonResult(BaseModel): + reward: float + category: StepRewardCategory + + +class ToolCallComparatorConfig(BaseModel): + word_count_similarity_threshold: float + floating_point_comparison_threshold: float = 1e-6 + + # Master switch: does the NUMBER of tool calls the model made affect its reward? + # + # False (default) turns parallel tool-call rewarding off entirely. The question asked of a + # response is only "did it make the expected call(s)?", so surplus calls are never penalized. + # This reproduces the behaviour that predates parallel tool-call support, which is why it is the + # default: existing datasets score exactly as they did before. It is also the honest default, + # because chat templates do not render differently for `parallel_tool_calls` (the Nemotron + # template never references the flag), so a model is never told how many calls it may make. + # + # True turns it on: the call count becomes part of the verdict, and the three settings below + # decide which counts are admissible and how partial matches are scored. Turn this on for + # datasets that use `expected_action.type: function_call_batch`. + parallel_tool_call_rewarding: bool = False + + # The three settings below are only consulted when `parallel_tool_call_rewarding` is True. + + # Cardinality gate: whether a response that makes fewer / more tool calls than expected is + # admissible at all. Both default to False, so the call count must match exactly. + allow_subset: bool = False + allow_superset: bool = False + + # Scoring for responses that clear the gate. See `ActionComparator.compare_tool_calls`. + parallel_tool_call_reward_mode: ParallelToolCallRewardMode = ParallelToolCallRewardMode.BINARY_STRICT + + +def find_maximum_matching(candidates: list[list[int]]) -> dict[int, int]: + """Maximum bipartite matching between expected and actual tool calls (Kuhn's algorithm). + + `candidates[expected_index]` lists the actual-call indices that expected call could match, and the + returned mapping goes from actual index to the expected index it was matched with. Greedy pairing + is not enough here: argument matching is a fuzzy relation, so one actual call can satisfy several + expected calls and an early arbitrary pairing can strand a later expected call that had no + alternative. Augmenting paths undo those pairings and recover the true maximum. + """ + matching: dict[int, int] = {} + + # Matching the most constrained expected calls first keeps the number of augmenting paths small. + for expected_index in sorted(range(len(candidates)), key=lambda index: len(candidates[index])): + _augment_matching(expected_index, candidates, matching, set()) + + return matching + + +def _augment_matching( + expected_index: int, + candidates: list[list[int]], + matching: dict[int, int], + visited_actual_indices: set[int], +) -> bool: + for actual_index in candidates[expected_index]: + if actual_index in visited_actual_indices: + continue + + visited_actual_indices.add(actual_index) + is_unmatched = actual_index not in matching + if is_unmatched or _augment_matching(matching[actual_index], candidates, matching, visited_actual_indices): + matching[actual_index] = expected_index + return True + + return False + + +class ActionComparator(BaseModel): + config: ToolCallComparatorConfig + + def compare_action(self, expected_action: ExpectedAction, actual_action: ExpectedAction) -> ActionComparisonResult: + match expected_action: + case MessageAction(): + # Currently, any chat message is assigned a reward of one. + if isinstance(actual_action, MessageAction): + return ActionComparisonResult(reward=1.0, category=StepRewardCategory.EXPECTED_CHAT_MESSAGE_FOUND) + + return ActionComparisonResult(reward=0.0, category=StepRewardCategory.NO_EXPECTED_CHAT_MESSAGE) + + case FunctionCallAction() | FunctionCallBatchAction(): + if isinstance(actual_action, MessageAction): + return ActionComparisonResult(reward=0.0, category=StepRewardCategory.NO_EXPECTED_TOOL_CALL) + + return self.compare_tool_calls(get_tool_calls(expected_action), get_tool_calls(actual_action)) + + case _: + raise NotImplementedError(f"Unsupported expected action: {expected_action!r}") + + def compare_tool_calls( + self, expected_calls: list[FunctionCallAction], actual_calls: list[FunctionCallAction] + ) -> ActionComparisonResult: + """Score a set of tool calls against the expected set, ignoring the order they were emitted in. + + `parallel_tool_call_rewarding` is the master switch. While it is False (the default) the number + of calls is simply not part of the verdict: the response is asked only whether it made the + expected call(s), and surplus calls cost nothing. That reproduces the behaviour that predates + parallel tool-call support. + + Turning it on makes the call count matter, in two independent stages. The cardinality gate + (`allow_subset` / `allow_superset`) decides whether a response that under- or over-calls is + admissible at all; anything it rejects scores zero. `parallel_tool_call_reward_mode` then decides + how much credit an admissible response earns: + + - `binary_strict` — 1.0 only if every required call matched, else 0.0. + - `fractional` — the matched fraction of the required calls. Surplus calls permitted by + `allow_superset` are free, so this rewards recall but not precision. + - `f1` — the harmonic mean of precision and recall, `2 * matched / (expected + actual)`. Missing + and surplus calls are penalized symmetrically, so a response only reaches 1.0 by matching the + expected calls exactly. + """ + expected_count = len(expected_calls) + actual_count = len(actual_calls) + + if expected_count == 1 and actual_count == 1: + # Preserve the single-call categories that predate parallel tool-call support. + return self.compare_tool_call(expected_calls[0], actual_calls[0]) + + if not self.is_call_count_admissible(expected_count, actual_count): + return ActionComparisonResult(reward=0.0, category=StepRewardCategory.FUNCTION_CALL_BATCH_LENGTH_DIFFERENT) + + candidates, failure_categories = self.build_match_candidates(expected_calls, actual_calls) + matching = find_maximum_matching(candidates) + reward = self.score_matched_calls(len(matching), expected_count, actual_count) + if reward == 1.0: + # A single expected call keeps the category it had before parallel support existed. + category = ( + StepRewardCategory.EXPECTED_TOOL_CALL + if expected_count == 1 + else StepRewardCategory.EXPECTED_TOOL_CALL_BATCH + ) + return ActionComparisonResult(reward=reward, category=category) + + category = self.resolve_failure_category( + matched_expected_indices=set(matching.values()), + failure_categories=failure_categories, + expected_count=expected_count, + actual_count=actual_count, + ) + return ActionComparisonResult(reward=reward, category=category) + + def is_call_count_admissible(self, expected_count: int, actual_count: int) -> bool: + if actual_count == expected_count: + return True + + # With parallel tool-call rewarding off, the call count is not part of the verdict at all. + if not self.config.parallel_tool_call_rewarding: + return True + + if actual_count < expected_count: + return self.config.allow_subset + + return self.config.allow_superset + + def required_match_count(self, expected_count: int, actual_count: int) -> int: + """How many expected calls must match for full credit under `binary_strict` and `fractional`.""" + if self.config.allow_subset: + return min(expected_count, actual_count) + + return expected_count + + def score_matched_calls(self, matched_count: int, expected_count: int, actual_count: int) -> float: + if not self.config.parallel_tool_call_rewarding: + # The call count is not part of the verdict, so neither the gate nor the reward mode + # applies: full credit exactly when every expected call was matched. + return 1.0 if matched_count == expected_count else 0.0 + + if self.config.parallel_tool_call_reward_mode == ParallelToolCallRewardMode.F1: + total_count = expected_count + actual_count + return 2 * matched_count / total_count if total_count else 0.0 + + required_count = self.required_match_count(expected_count, actual_count) + if self.config.parallel_tool_call_reward_mode == ParallelToolCallRewardMode.FRACTIONAL: + return matched_count / required_count if required_count else 0.0 + + return 1.0 if matched_count == required_count else 0.0 + + def build_match_candidates( + self, expected_calls: list[FunctionCallAction], actual_calls: list[FunctionCallAction] + ) -> tuple[list[list[int]], list[StepRewardCategory]]: + """Pair every expected call with the actual calls it matches, keeping why the others missed.""" + candidates: list[list[int]] = [] + failure_categories: list[StepRewardCategory] = [] + + for expected_call in expected_calls: + matching_actual_indices: list[int] = [] + failure_category = StepRewardCategory.UNEXPECTED_TOOL + + for actual_index, actual_call in enumerate(actual_calls): + result = self.compare_tool_call(expected_call, actual_call) + if result.reward == 1.0: + matching_actual_indices.append(actual_index) + + elif failure_category == StepRewardCategory.UNEXPECTED_TOOL: + # Keep the first reason that got past the tool name; it explains the closest near miss. + failure_category = result.category + + candidates.append(matching_actual_indices) + failure_categories.append(failure_category) + + return candidates, failure_categories + + def resolve_failure_category( + self, + matched_expected_indices: set[int], + failure_categories: list[StepRewardCategory], + expected_count: int, + actual_count: int, + ) -> StepRewardCategory: + unmatched_expected_indices = [ + expected_index + for expected_index in range(expected_count) + if expected_index not in matched_expected_indices + ] + + # Either every expected call was matched and the only defect left is surplus calls, or every + # actual call was consumed and the only defect left is missing calls. Reporting an argument + # mismatch in those cases would point at a comparison that is not why the reward was docked. + if not unmatched_expected_indices or len(matched_expected_indices) == actual_count: + return StepRewardCategory.FUNCTION_CALL_BATCH_LENGTH_DIFFERENT + + return failure_categories[unmatched_expected_indices[0]] + + def compare_tool_call( + self, expected_tool_call: FunctionCallAction, actual_tool_call: FunctionCallAction + ) -> ActionComparisonResult: + if expected_tool_call.name != actual_tool_call.name: + return ActionComparisonResult(reward=0.0, category=StepRewardCategory.UNEXPECTED_TOOL) + + # It is assumed that the expected arguments string is a string representation of a JSON object. + expected_arguments = json.loads(expected_tool_call.arguments) + + try: + actual_arguments = json.loads(actual_tool_call.arguments) + except (JSONDecodeError, UnicodeDecodeError): + return ActionComparisonResult(reward=0.0, category=StepRewardCategory.ARGUMENTS_DECODE_ERROR) + + arguments_match, category = self.compare_tool_call_arguments(expected_arguments, actual_arguments) + if arguments_match: + return ActionComparisonResult(reward=1.0, category=StepRewardCategory.EXPECTED_TOOL_CALL) + + return ActionComparisonResult(reward=0.0, category=category) + + def compare_tool_call_arguments( + self, expected_value: Any, actual_value: Any + ) -> tuple[bool, Optional[StepRewardCategory]]: + if not isinstance(actual_value, type(expected_value)): + return False, StepRewardCategory.ARGUMENT_VALUE_TYPE_DIFFERENT + + if isinstance(expected_value, dict): + if set(expected_value.keys()) != set(actual_value.keys()): + return False, StepRewardCategory.ARGUMENT_OBJECT_KEYS_DIFFERENT + + for expected_dict_key, expected_dict_value in expected_value.items(): + actual_dict_value = actual_value[expected_dict_key] + dict_value_match, dict_value_category = self.compare_tool_call_arguments( + expected_dict_value, actual_dict_value + ) + if not dict_value_match: + return dict_value_match, dict_value_category + + return True, None + + elif isinstance(expected_value, list): + if len(expected_value) != len(actual_value): + return False, StepRewardCategory.ARGUMENT_LIST_LENGTH_DIFFERENT + + for expected_list_element, actual_list_element in zip(expected_value, actual_value): + list_element_match, list_element_category = self.compare_tool_call_arguments( + expected_list_element, actual_list_element + ) + if not list_element_match: + return list_element_match, list_element_category + + return True, None + + elif isinstance(expected_value, float): + if abs(actual_value - expected_value) < self.config.floating_point_comparison_threshold: + return True, None + else: + return False, StepRewardCategory.ARGUMENT_VALUE_DIFFERENT + + elif isinstance(expected_value, str): + # For now, strings are compared by using whitespace to split them into lower-case + # words, counting the words, and comparing the word counts using Jaccard similarity. + expected_word_counts = Counter(expected_value.strip().lower().split()) + actual_word_counts = Counter(actual_value.strip().lower().split()) + expected_word_total = expected_word_counts.total() + actual_word_total = actual_word_counts.total() + + if expected_word_total < 2 or actual_word_total < 2: + if expected_value != actual_value: + return False, StepRewardCategory.ARGUMENT_VALUE_DIFFERENT + + else: + intersection_word_counts = expected_word_counts & actual_word_counts + + word_count_similarity = intersection_word_counts.total() / (expected_word_total + actual_word_total) + if word_count_similarity < self.config.word_count_similarity_threshold: + return False, StepRewardCategory.ARGUMENT_VALUE_DIFFERENT + + return True, None + + elif expected_value == actual_value: + return True, None + + else: + return False, StepRewardCategory.ARGUMENT_VALUE_DIFFERENT diff --git a/app/version.py b/app/version.py index ce71e6611aaeccd94f2d026dd0b334660bc85b10..22894d3eca970c7f4171b18480cbd41da7454970 100644 --- a/app/version.py +++ b/app/version.py @@ -26,9 +26,9 @@ def app_version() -> str: @lru_cache(maxsize=1) def source_hash() -> str: h = hashlib.sha256() - for base in (ROOT / "app", ROOT / "web" / "js"): + for base in (ROOT / "app", ROOT / "web", ROOT / "web-admin"): for p in sorted(base.rglob("*")): - if p.is_file() and p.suffix in (".py", ".js", ".txt", ".json") and "__pycache__" not in p.parts: + if p.is_file() and p.suffix in (".py", ".js", ".txt", ".json", ".css", ".html") and "__pycache__" not in p.parts: h.update(p.relative_to(ROOT).as_posix().encode()) h.update(p.read_bytes()) return h.hexdigest()[:12] diff --git a/pyproject.toml b/pyproject.toml index 6091666528c362c5fcb42a93124619f009df0938..895aed1076536a7131af1e99bea0ccb1ff8c8a57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "httpx>=0.27", "huggingface_hub>=1.29", "pyyaml>=6", + "packaging>=24.2", "cryptography>=44", "openenv", "harbor>=0.23", diff --git a/scripts/deploy_spaces.py b/scripts/deploy_spaces.py index e72aa0f21aa073802251e8fec7c0bdf306ae6647..43622eb6ed8f3506d7062a07d4a391b2a0245b26 100644 --- a/scripts/deploy_spaces.py +++ b/scripts/deploy_spaces.py @@ -42,8 +42,8 @@ SPACES = { } # never uploaded: local data and caches, the READMEs (each Space gets its own as README.md), editor and OS files IGNORE = [".venv/*", ".local-data/*", "*/__pycache__/*", "__pycache__/*", "*.pyc", ".pytest_cache/*", ".ruff_cache/*", - ".gradio/*", "node_modules/*", ".DS_Store", "*/.DS_Store", "README.md", "SPACE_README.md", "ADMIN_SPACE_README.md", - ".env", "*.env", ".git/*"] + ".gradio/*", "node_modules/*", ".DS_Store", "*/.DS_Store", "README.md", "SPACE_README.md", "ADMIN_SPACE_README.md", "EXPLORER_AUDIT.md", + ".env", ".env.*", "*/.env", "*/.env.*", "*.env", "*.pem", "*.key", ".git/*"] SKIP_DIRS = {".venv", ".local-data", "__pycache__", ".pytest_cache", ".ruff_cache", ".gradio", "node_modules", ".git"} @@ -53,10 +53,10 @@ def files() -> list[str]: """What goes into the Space: walked without descending into local data, caches or the virtualenv.""" out = [] for root, dirs, names in os.walk(ROOT): - dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS) + dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS and not (Path(root) / d).is_symlink()) for n in sorted(names): rel = (Path(root) / n).relative_to(ROOT).as_posix() - if not any(fnmatch.fnmatch(rel, pat) for pat in IGNORE): + if not (Path(root) / n).is_symlink() and not any(fnmatch.fnmatch(rel, pat) for pat in IGNORE): out.append(rel) return out diff --git a/tests/reward-stats.test.mjs b/tests/reward-stats.test.mjs new file mode 100644 index 0000000000000000000000000000000000000000..841e34ea29d22d29a1645eedd75ab7b6ee6e44d3 --- /dev/null +++ b/tests/reward-stats.test.mjs @@ -0,0 +1,20 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { rewardDistribution } from '../web/js/reward-stats.js'; + +test('negative and unbounded rewards remain in the histogram', () => { + const d = rewardDistribution([-100, -5, 0, 0.5, 1, 500]); + assert.equal(d.min, -100); + assert.equal(d.max, 500); + assert.equal(d.hist.reduce((a, b) => a+b), 6); + assert.equal(d.hist[0], 1); + assert.equal(d.hist[9], 1); +}); +test('empty and normalized rewards keep a useful finite range', () => { + for (const xs of [[], [0, 0.5, 1], [NaN, Infinity]]) { + const d = rewardDistribution(xs); + assert.equal(d.min, 0); + assert.equal(d.max, 1); + assert.ok(d.hist.every(Number.isFinite)); + } +}); diff --git a/tests/schemaform.test.mjs b/tests/schemaform.test.mjs new file mode 100644 index 0000000000000000000000000000000000000000..8f36fd947a9fd0073e591763e99b48a805b236a4 --- /dev/null +++ b/tests/schemaform.test.mjs @@ -0,0 +1,89 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { schemaForm, readForm, resolveSchema, validateValue } from '../web/js/schemaform.js'; + +const form = (fields = [], root = null) => ({ + querySelector: () => root, + querySelectorAll: () => fields.map(([k, t, value]) => ({ dataset: { k, t }, value })), +}); +const plain = (v) => JSON.parse(JSON.stringify(v)); + +test('optional booleans can be omitted, and explicit false remains false', () => { + const schema = { type: 'object', properties: { enabled: { type: 'boolean', default: true } } }; + assert.deepEqual(plain(readForm(form([['enabled', 'bool', '']]), schema)), {}); + assert.deepEqual(plain(readForm(form([['enabled', 'bool', 'false']]), schema)), { enabled: false }); + assert.match(schemaForm(schema), /Use server default/); +}); + +test('integer inputs reject fractions rather than truncating actions', () => { + const s = { type: 'object', properties: { action: { type: 'integer', minimum: 0, maximum: 3 } }, required: ['action'] }; + for (const value of ['1.5', '4', '-1', 'Infinity', '9007199254740992', '']) { + assert.throws(() => readForm(form([['action', 'int', value]]), s)); + } + assert.equal(readForm(form([['action', 'int', '0']]), s).action, 0); +}); + +test('enum values retain their JSON type and never silently pick the first option', () => { + const s = { type: 'object', properties: { action: { enum: [0, 1, 'stop', null] } }, required: ['action'] }; + assert.match(schemaForm(s), /Choose a value/); + assert.equal(readForm(form([['action', 'enum', '0']]), s).action, 0); + assert.equal(readForm(form([['action', 'enum', 'null']]), s).action, null); + assert.throws(() => readForm(form([['action', 'enum', '"bad"']]), s)); +}); + +test('local references retain defaults and referenced discriminators', () => { + const s = { type: 'object', $defs: { Mode: { const: 'move' }, Count: { type: 'integer', default: 7 } }, + properties: { type: { $ref: '#/$defs/Mode' }, count: { $ref: '#/$defs/Count' } } }; + assert.match(schemaForm(s), /value="7"/); + assert.equal(readForm(form(), s).type, 'move'); +}); + +test('recursive and missing references are bounded', () => { + const s = { $ref: '#/$defs/Loop', $defs: { Loop: { $ref: '#/$defs/Loop' } } }; + assert.doesNotThrow(() => schemaForm(s)); + assert.deepEqual(resolveSchema({ $ref: '#/$defs/missing' }, s), {}); +}); + +test('root action unions use JSON without losing nested arguments', () => { + const s = { oneOf: [ + { type: 'object', properties: { type: { const: 'move' }, position: { type: 'array', items: { type: 'number' }, minItems: 2 } }, required: ['type', 'position'] }, + { type: 'object', properties: { type: { const: 'stop' } }, required: ['type'] }, + ] }; + assert.match(schemaForm(s), /data-root-json/); + assert.deepEqual(readForm(form([], { value: '{"type":"move","position":[1,2]}' }), s), { type: 'move', position: [1, 2] }); + assert.throws(() => readForm(form([], { value: '{"type":"move"}' }), s)); +}); + +test('open objects have a JSON editor rather than claiming no inputs', () => { + assert.match(schemaForm({ type: 'object' }), /data-root-json/); + assert.match(schemaForm({ type: 'object', additionalProperties: false }), /No inputs/); +}); + +test('all interpolated upstream schema values are escaped', () => { + const payload = '\">'; + const s = { type: 'object', properties: { [payload]: { type: 'number', default: payload, minimum: payload } } }; + const html = schemaForm(s, { prefix: payload }); + assert.ok(!html.includes(' x[1]); + assert.equal(ids.length, new Set(ids).size); +}); + +test('prototype-like field names remain ordinary JSON properties', () => { + const schema = JSON.parse('{"type":"object","properties":{"__proto__":{"type":"object"}}}'); + const result = readForm(form([['__proto__', 'json', '{"polluted":true}']]), schema); + assert.equal(Object.getPrototypeOf(result), null); + assert.equal({}.polluted, undefined); + assert.equal(JSON.parse(JSON.stringify(result)).__proto__.polluted, true); +}); + +test('nested required, size and additional-properties constraints report actionable errors', () => { + const s = { type: 'object', properties: { args: { type: 'object', properties: { text: { type: 'string', minLength: 2 } }, required: ['text'], additionalProperties: false } } }; + assert.throws(() => validateValue({ args: {} }, s), /args.text is required/); + assert.throws(() => validateValue({ args: { text: '' } }, s), /at least 2/); + assert.throws(() => validateValue({ args: { text: 'ok', answer: 'secret' } }, s), /not an allowed field/); +}); diff --git a/tests/space-media.test.mjs b/tests/space-media.test.mjs new file mode 100644 index 0000000000000000000000000000000000000000..b08f86ed9aa839165ffd3f5b3f7f09ea8fdf8ef6 --- /dev/null +++ b/tests/space-media.test.mjs @@ -0,0 +1,24 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spaceMedia } from '../web/js/space-media.js'; +import { taskContent } from '../web/js/space-task-view.js'; + +test('lazy Space media resolves against the serving Space without mutating observations', () => { + const raw = { observation: { asset_path: '/assets/hash?task_id=a', mime: 'audio/wav' } }; + const out = spaceMedia(raw, 'https://owner-env.hf.space'); + assert.equal(out.observation.media_url, 'https://owner-env.hf.space/assets/hash?task_id=a'); + assert.equal(raw.observation.media_url, undefined); +}); + +test('media paths cannot change origin or use active URL schemes', () => { + for (const path of ['//evil.example/assets/a', 'javascript:alert(1)', 'https://evil.example/a', '/other/a']) + assert.equal(spaceMedia({ asset_path: path, mime: 'audio/wav' }, 'https://owner-env.hf.space').media_url, undefined); + assert.equal(spaceMedia({ asset_path: '/assets/a', mime: 'audio/wav' }, 'http://localhost').media_url, undefined); +}); + +test('task inputs are escaped and lazy media remains an available episode', () => { + const html = taskContent({ prompt: '', task_id: 'a'.repeat(200), media_ready: false }); + assert.ok(!html.includes('= at + auto_index.cycle() + assert len(called) == 1 # app restarts and duplicate workers honor cadence + assert auto_index.cached_listing(at + 3601) == (None, None) + assert auto_index.cached_listing(at - 1) == (None, None) + + +def test_locked_worker_cannot_publish_duplicate_snapshot(monkeypatch): + fcntl = pytest.importorskip("fcntl") + config.CACHE_DIR.mkdir() + monkeypatch.setattr(indexer, "run", lambda *a, **k: pytest.fail("duplicate indexer")) + with (config.CACHE_DIR / "auto-index.lock").open("a") as lease: + fcntl.flock(lease, fcntl.LOCK_EX | fcntl.LOCK_NB) + auto_index.cycle() + assert auto_index.status() == {} + + +def test_failure_preserves_published_pointer_and_backs_off(monkeypatch): + store = snapshot.open_store() + store.put_bytes(snapshot.POINTER, b"last-good-snapshot") + def fail(*a, **k): + raise ConnectionError("upstream response must not enter public status") + monkeypatch.setattr(indexer, "run", fail) + auto_index.cycle() + first = auto_index.status() + assert first["state"] == "retrying" and first["error"] == "ConnectionError" + assert first["next_at"] > time.time() + 590 + assert store.read_bytes(snapshot.POINTER) == b"last-good-snapshot" + auto_index._save({**first, "next_at": 0}) + auto_index.cycle() + assert auto_index.status()["failures"] == 2 + assert auto_index.status()["next_at"] > time.time() + 1190 + + +def test_missing_or_partial_listing_requires_fresh_discovery(): + assert auto_index.cached_listing(time.time()) == (None, None) + doc = {"at": time.time(), "full": False, "rows": [{"id": "owner/partial"}]} + (config.STORAGE_DIR / "listing.json.gz").write_bytes(gzip.compress(json.dumps(doc).encode())) + assert auto_index.cached_listing(time.time()) == (None, None) + + +def test_shutdown_terminates_only_owned_indexer_child(monkeypatch): + calls = [] + class Child: + def poll(self): return None + def terminate(self): calls.append("terminate") + def wait(self, **kwargs): calls.append("wait") + def popen(command, **kwargs): + assert command[-2:] == ["-m", "app.auto_index"] + assert kwargs["env"]["HF_HUB_DISABLE_IMPLICIT_TOKEN"] == "1" + return Child() + monkeypatch.setattr(auto_index.subprocess, "Popen", popen) + auto_index._stop.set() + try: + auto_index._child() + finally: + auto_index._stop.clear() + assert calls == ["terminate", "wait"] diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py new file mode 100644 index 0000000000000000000000000000000000000000..ac5634b93f7a7e294bb9ffc9b59760eb01ed8476 --- /dev/null +++ b/tests/test_capabilities.py @@ -0,0 +1,68 @@ +"""Support claims reflect runtime evidence, not a source's framework tag.""" +import pytest + +from app import spaces_live as live +from app.envs.capabilities import dataset_support, space_support + + +def state(support, feature): + return next(c["state"] for c in support["capabilities"] if c["id"] == feature) + + +@pytest.mark.parametrize("framework,runner", [("Verifiers", "verifiers"), ("NeMo Gym", "nemo-gym"), ("verl", "verl"), ("SkyRL", "skyrl")]) +def test_native_framework_dataset_does_not_claim_local_execution(framework, runner): + support = dataset_support(framework, options=[{"runner": runner, "label": framework, "ok": False, "why": "Use its native harness"}]) + assert state(support, "browse") == "available" + assert state(support, "run") == "external" + assert support["framework"]["docs"].startswith("https://") + + +def test_task_support_distinguishes_blocked_from_runnable(): + options = [{"runner": "harbor", "label": "Harbor", "ok": False, "why": "GPU required"}] + assert state(dataset_support("Harbor", options=options), "run") == "unavailable" + options[0]["ok"] = True + assert state(dataset_support("Harbor", options=options), "run") == "available" + assert state(dataset_support("Harbor"), "run") == "unknown" + + +def test_sleeping_server_never_claims_live_capabilities(): + s = space_support({"running": False, "stage": "SLEEPING", "last_seen": {"step_api": True}}) + assert state(s, "play") == "unknown" + assert s["transport"] is None + + +@pytest.mark.parametrize("protocol", ["openenv", "gymnasium", "nemo-gym", "mcp", "api"]) +def test_probe_uses_published_protocol_not_reset_route_or_tag(monkeypatch, protocol): + from app import catalog + catalog._memo.clear() + rec = {"id": "fixture/env", "stage": "RUNNING", "host": "https://fixture-env.hf.space", "base_path": None, "tags": ["openenv"]} + monkeypatch.setattr(live, "record", lambda *a, **k: rec) + paths = {"/reset": {"post": {}}, "/step": {"post": {}}} if protocol in {"openenv", "gymnasium"} else ( + {"/seed_session": {"post": {}}, "/verify": {"post": {}}} if protocol == "nemo-gym" else {"/mcp": {"post": {}}} if protocol == "mcp" else {"/guess": {"post": {}}}) + docs = {"/openapi.json": {"info": {"version": "1"}, "paths": paths}} + if protocol == "openenv": + docs["/schema"] = {k: {"type": "object", "properties": {}} for k in ("action", "observation", "state")} + monkeypatch.setattr(live, "_get_json", lambda rec, path: docs.get(path)) + monkeypatch.setattr(live, "_ui_path", lambda rec: None) + monkeypatch.setattr(live, "_tools", lambda rec: ([{"name": "guess"}] if protocol == "mcp" else [], None, {})) + monkeypatch.setattr(live, "remember", lambda *a: None) + info = live.probe("fixture/env") + assert info["framework"] == protocol + assert info["openenv"] is (protocol == "openenv") + assert info["step_api"] is (protocol == "openenv") + if protocol == "gymnasium": + assert {r["path"] for r in info["api"]} == {"/reset", "/step"} + assert info["support"]["transport"] == "http" + assert state(info["support"], "play") == "available" + + +def test_private_dataset_does_not_offer_public_mcp(): + assert state(dataset_support("Harbor", restricted=True), "mcp") == "unavailable" + + +def test_packed_harbor_rows_keep_the_harbor_lifecycle(): + assert dataset_support("Harbor (packed)")["framework"]["id"] == "harbor" + + +def test_unknown_runner_is_not_claimed_as_hosted(): + assert state(dataset_support("Custom", options=[{"runner": "custom", "label": "Custom", "ok": True}]), "run") == "unavailable" diff --git a/tests/test_community.py b/tests/test_community.py new file mode 100644 index 0000000000000000000000000000000000000000..4db4f3d4180da2ede40a900eb5a833e811b32b52 --- /dev/null +++ b/tests/test_community.py @@ -0,0 +1,96 @@ +import json + +import pytest +from fastapi.testclient import TestClient + +from app import auth, community, main, settings, store + +client = TestClient(main.app) + + +def record(id, runner="harbor", **fields): + return {"id": id, "dataset": f"org/{runner}", "path": "task/1", "title": "A task", "user": "hidden-user", + "runner": runner, "status": "done", "reward": 1, "visibility": "public", "model": "org/model", + "created_at": 10, **fields} + + +@pytest.fixture +def records(monkeypatch): + rows = [record("harbor", reward=-3, dataset="a/same"), record("mimo", "mimo", domain="code", reward=0.25), + record("nemo", "nemo-gym", reward=1, dataset="b/same"), record("future", "custom-runtime", reward=20), + record("private", visibility="private"), record("restricted", restricted=True), + record("failed", status="failed", reward=None), record("hidden")] + monkeypatch.setattr(store, "list_runs", lambda **kw: rows) + monkeypatch.setattr(store, "has_artifact", lambda *a: False) + monkeypatch.setattr(settings, "get", lambda key, default=None: ["hidden"] if key == "hidden_runs" else default) + return rows + + +def test_all_saved_runners_share_the_feed_with_scoped_counts(records): + data = client.get("/api/community").json() + assert {r["id"] for r in data["runs"]} == {"harbor", "mimo", "nemo", "future"} + assert data["total"] == data["stats"]["runs"] == 4 + assert data["stats"]["datasets"] == data["stats"]["runners"] == 4 + assert not data["stats"]["comparable"] + assert "mean" not in data["stats"] # no meaningless global reward average + assert {r["group"] for r in data["runs"]} >= {"a/same", "b/same"} + assert "hidden-user" not in json.dumps(data) + for runner in ("harbor", "mimo", "nemo-gym", "custom-runtime"): + subset = client.get("/api/community", params={"runner": runner}).json() + assert subset["total"] == subset["stats"]["runs"] == subset["stats"]["datasets"] == 1 + assert {r["runner"] for r in subset["runs"]} == {runner} + assert len(subset["facets"]["runner"]) == 4 # own facet retains alternatives + assert len(subset["facets"]["dataset"]) == 1 + + +def test_search_includes_environment_runner_and_model_and_empty_counts(records): + for query, expected in (("b/same", "nemo"), ("nemo-gym", "nemo"), ("org/mimo model", "mimo")): + data = client.get("/api/community", params={"q": query}).json() + assert [r["id"] for r in data["runs"]] == [expected] + empty = client.get("/api/community?dataset=nobody/missing").json() + assert empty["total"] == empty["stats"]["datasets"] == empty["stats"]["models"] == empty["stats"]["runners"] == 0 + assert empty["stats"]["results"] == [] + + +def test_reward_summaries_separate_metrics_domains_judges_and_revisions(): + rows = [record("a", reward=-4), record("b", reward=-2)] + for field, value in (("reward_key", "latency"), ("domain", "code"), ("judge", "judge/new"), ("sha", "new-revision")): + rows.append(record(field, reward=100, **{field: value})) + data = community.stats(rows) + assert len(data["results"]) == 5 + original = next(r for r in data["results"] if r["runs"] == 2) + assert original["mean"] == -3 and original["max"] == -2 + assert community.stats([record("a", reward=1e308), record("b", reward=1e308)])["results"][0]["mean"] == 1e308 + + +@pytest.mark.parametrize("value", [None, "1", True, {}, float("nan"), float("inf"), 10**1000]) +def test_invalid_rewards_cannot_break_or_enter_public_feed(value, monkeypatch): + monkeypatch.setattr(settings, "get", lambda *a: []) + assert not main._shareable(record("bad", reward=value)) + + +def test_pagination_sort_and_negative_reward_band(records): + records.extend(record(f"r{i:02}", created_at=i, reward=-i, cost={"total": None}) for i in range(20)) + all_rows = client.get("/api/community?sort=reward_asc").json()["runs"] + pages = [client.get("/api/community", params={"sort": "reward_asc", "offset": i, "limit": 5}).json()["runs"] for i in range(0, 24, 5)] + assert [r["id"] for page in pages for r in page] == [r["id"] for r in all_rows] + negative = client.get("/api/community?reward=zero").json() + assert negative["total"] == 21 and all(r["reward"] <= 0 for r in negative["runs"]) + assert client.get("/api/community?sort=cost_asc").status_code == 200 + + +@pytest.mark.parametrize("runner,domain,events", [("harbor", None, False), ("mimo", "code", True), ("nemo-gym", "nemo-gym", True), (None, "code", True)]) +def test_trace_routes_match_runner_and_keep_public_scrubbing(monkeypatch, runner, domain, events): + from app.mimo.runner import core + r = record("trace", runner, domain=domain) + monkeypatch.setattr(store, "get", lambda id: r) + monkeypatch.setattr(settings, "get", lambda *a: []) + monkeypatch.setattr(auth, "current_user", lambda request: None) + monkeypatch.setattr(store, "read_artifact", lambda *a: b'[{"content":"hidden-user"}]') + monkeypatch.setattr(main.runner, "is_live", lambda *a: False) + monkeypatch.setattr(core, "live", lambda *a: None) + monkeypatch.setattr(core, "events", lambda *a: [{"content": "hidden-user"}]) + response = client.get("/api/runs/trace") + assert response.status_code == 200 and "hidden-user" not in response.text + assert ("events" in response.json()) == events + assert response.json()["run"]["runner"] == (runner or "mimo") diff --git a/tests/test_deploy_spaces.py b/tests/test_deploy_spaces.py new file mode 100644 index 0000000000000000000000000000000000000000..66a4d1d4208f3d5a562461ff79f3be4f53e24f23 --- /dev/null +++ b/tests/test_deploy_spaces.py @@ -0,0 +1,12 @@ +from scripts import deploy_spaces + + +def test_upload_manifest_excludes_local_secrets_and_symlinks(tmp_path, monkeypatch): + for name in ('app/main.py','web/app.css','.env','.env.production','sub/.env.local','sub/private.key','.local-data/run.json','.venv/env.py'): + path=tmp_path/name + path.parent.mkdir(parents=True,exist_ok=True) + path.write_text('sample') + (tmp_path/'linked.py').symlink_to(tmp_path/'.env') + (tmp_path/'linked-dir').symlink_to(tmp_path/'.local-data',target_is_directory=True) + monkeypatch.setattr(deploy_spaces,'ROOT',tmp_path) + assert deploy_spaces.files()==['app/main.py','web/app.css'] diff --git a/tests/test_discovery.py b/tests/test_discovery.py new file mode 100644 index 0000000000000000000000000000000000000000..d5bc684a6d645bc53dc36b75eb916dd7d17c3689 --- /dev/null +++ b/tests/test_discovery.py @@ -0,0 +1,69 @@ +"""Discovery evidence is metadata, not a runtime attestation or an editorial decision.""" +from types import SimpleNamespace + +import pytest +from huggingface_hub import SpaceInfo + +from app import catalog + + +def space(sid="org/env", *, tags=(), files=(), **kw): + return SpaceInfo(id=sid, sdk="docker", private=False, tags=list(tags), likes=0, + siblings=[{"rfilename": f} for f in files], **kw) + + +@pytest.mark.parametrize("files", [("openenv.yaml",), ("my_env/openenv.yml",)]) +def test_manifest_discovery(files): + d = catalog._space_summary(space(files=files)) + assert d["openenv"] and d["manifest"] == files[0] + + +@pytest.mark.parametrize("file", ["src/openenv.yaml", "templates/openenv.yaml", "examples/openenv.yaml", + "build/openenv.yaml", "vendor/openenv.yaml", "envs/echo/openenv.yaml", + "src/openenv/cli/templates/openenv.yaml", ".venv/openenv.yaml"]) +def test_vendored_manifests_are_not_declarations(file): + d = catalog._space_summary(space(files=[file])) + assert not d["openenv"] and d["manifest"] is None + + +@pytest.mark.parametrize("tag", ["openenv", "library:openenv", "openenv-0.7.0"]) +def test_tag_only_remains_a_candidate_without_a_manifest(tag): + d = catalog._space_summary(space(tags=[tag])) + assert d["openenv"] and d["manifest"] is None + + +def test_names_and_generic_rl_mcp_tags_do_not_establish_openenv(): + d = catalog._space_summary(space("openenv/not-an-env", tags=["rl-environment", "mcp-server"])) + assert not d["openenv"] and d["mcp"] + + +def test_space_census_deduplicates_and_applies_visibility_to_featured(monkeypatch): + good = space(tags=["openenv"]) + article = space("org/article", tags=["openenv", "research-article-template"]) + private = space("org/private"); private.private = True + static = space("org/static"); static.sdk = "static" + by = {s.id: s for s in [good, article, private, static]} + monkeypatch.setattr(catalog, "collections", lambda: [{"ids": [f"space:{s}" for s in by]}]) + monkeypatch.setattr(catalog, "_api", lambda: SimpleNamespace(list_spaces=lambda **kw: iter(by.values()), + space_info=lambda sid, **kw: by[sid])) + assert [r["id"] for r in catalog.spaces()] == [good.id] + + +def test_failed_full_census_is_not_silently_published(monkeypatch): + def listing(**kw): + yield space(tags=["openenv"]) + raise ConnectionError("page two unavailable") + + monkeypatch.setattr(catalog, "collections", lambda: []) + monkeypatch.setattr(catalog, "_api", lambda: SimpleNamespace(list_spaces=listing)) + with pytest.raises(ConnectionError): + catalog.spaces(full=True) + assert len(catalog.spaces(full=False)) == 1 # a quick bootstrap may serve candidates while refreshing + + +def test_collection_never_reclassifies_framework(monkeypatch): + d = catalog._space_summary(space(tags=["ors"])) + monkeypatch.setattr(catalog, "_listing_rows", lambda: [d]) + monkeypatch.setattr(catalog, "collections", lambda: [{"id": "fineenvs", "ids": [d["key"]]}]) + monkeypatch.setattr(catalog, "hidden", lambda: set()) + assert catalog.environments()[0]["framework"] == "ors" diff --git a/tests/test_frameworks.py b/tests/test_frameworks.py index 7b1dd5dc4380815ca3a4283b6b31eabe01aadf09..0c317007284f7c810a2f56d5bb1c0c8bcdbd314a 100644 --- a/tests/test_frameworks.py +++ b/tests/test_frameworks.py @@ -135,7 +135,9 @@ def test_nemo_gym_pivot_row_reads_as_a_conversation_and_hides_the_experts_next_s assert "--agent toolcall_schema_single_step_tool_use_with_argument_comparison_agent" in run and "gym eval run --no-serve" in run assert "--repo-id nvidia/Nemotron-RL-Agentic-Function-Calling-Pivot-v1" in run and "sed -n '1p'" in run opt = view["framework_run"] - assert opt["runner"] == "nemo-gym" and opt["ok"] is False and "NeMo Gym" in opt["why"] + # This browsing fixture mixes Chat Completions and Responses tool formats; + # execution requires the dataset's native Responses function definitions. + assert opt["runner"] == "nemo-gym" and opt["ok"] is False and "native NeMo Gym" in opt["why"] assert ["Step", "turn 6 · step 1"] in view["glance"] and card["chips"][0] == "turn 6 · step 1" @@ -210,6 +212,8 @@ def test_nemo_gym_blends_route_each_row_and_run_without_one_agent(): def test_nemo_gym_catalog_is_current_and_complete(): cat = nemogym.catalog() assert len(cat["commit"]) == 40 and len(cat["agents"]) > 100 and len(cat["servers"]) > 100 + assert cat["servers"]["instruction_following"]["verification"] == "Rule-based instruction checks" + assert cat["servers"]["math_proof_judgement"]["verification"] == "Deterministic answer parsing" for name in ("mcqa_simple_agent", "workplace_assistant_simple_agent", "calendar_simple_agent", "code_gen_simple_agent", "instruction_following_simple_agent", "reasoning_gym_simple_agent"): a = cat["agents"][name] @@ -406,3 +410,11 @@ def test_search_never_finds_a_row_through_its_withheld_answer(monkeypatch): files = rows.Files.__new__(rows.Files) monkeypatch.setattr(rows.Files, "_scan", lambda self, c, s, o, match, what: ({"rows": [h for h in found if match(h[1])]}, None)) assert [i for i, _, _ in files.search("d", "train", "7", 0)[0]["rows"]] == [] + + +def test_verifiers_v1_taskdata_system_prompt_is_part_of_the_conversation(): + row = {"prompt": "Solve the task", "system_prompt": "Use the supplied tools", "answer": "PRIVATE_REFERENCE"} + view, _, _, _, text = everything(verifiers_rows.Verifiers(), row, tags=["verifiers"]) + turns = next(s for s in view["sections"] if s["id"] == "task")["body"][0]["data"]["turns"] + assert [(t["role"], t["text"]) for t in turns] == [("system", "Use the supplied tools"), ("user", "Solve the task")] + assert "PRIVATE_REFERENCE" not in text diff --git a/tests/test_live.py b/tests/test_live.py index bf96695e41bf179289e8bc4e22625ba216479d20..91ff9d8ded66d6e1cf4d66ceed1a570b60bdd99c 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -352,7 +352,7 @@ OPENAPI = {"paths": { def test_routes_are_plain_paths_on_non_openenv_servers_only(): routes = live._api_routes(OPENAPI, openenv=False) - assert {(r["method"], r["path"]) for r in routes} == {("POST", "/guess"), ("GET", "/tasks/{task_id}"), ("POST", "/self")} + assert {(r["method"], r["path"]) for r in routes} == {("POST", "/guess"), ("GET", "/tasks/{task_id}"), ("POST", "/self"), ("POST", "/reset")} guess = next(r for r in routes if r["path"] == "/guess") assert guess["schema"]["properties"]["word"] == {"type": "string", "title": "Word"} and guess["schema"]["required"] == ["word"] assert next(r for r in routes if r["path"] == "/tasks/{task_id}")["params"] == ["task_id"] @@ -383,8 +383,8 @@ def test_route_calls_keep_their_cookies_and_withhold_answers_from_gets(monkeypat return httpx.Response(200, json={"output": "ok"}, headers={"set-cookie": "session=abc; path=/"}) return httpx.Response(200, json={"task_id": "t1", "prompt": "p", "answer": "SECRET", "meta": {"ground_truth": "TRUTHVAL"}}) - real = httpx.Client - monkeypatch.setattr(live.httpx, "Client", lambda **kw: real(transport=httpx.MockTransport(handler), **kw)) + real = live.SpaceClient + monkeypatch.setattr(live, "SpaceClient", lambda **kw: real(transport=httpx.MockTransport(handler), **kw)) sid = live.start("org/env", "web:a")["session"] out = live.act(sid, "web:a", "http", {"method": "POST", "path": "/guess", "body": {"word": "crane"}})["result"] assert out["status"] == 200 and out["json"] == {"output": "ok"} @@ -397,7 +397,7 @@ def test_route_calls_keep_their_cookies_and_withhold_answers_from_gets(monkeypat def test_a_reset_step_server_still_lists_its_tools(monkeypatch): """No MCP tools of its own (OpenEnv answers "Environment does not support MCP"): the page gets exactly what the MCP - bridge offers an agent, step taking the server's own action fields. A tool server gets nothing added.""" + bridge offers an agent, step taking the server's own action fields. Tool servers use the same descriptor.""" info = {"running": True, "stage": "RUNNING", "step_api": True, "mcp": None, "task_api": None, "schema": {"action": {"type": "object", "properties": {"latex": {"type": "string", "description": "Predicted LaTeX"}}, "required": ["latex"]}}} monkeypatch.setattr(live, "probe", lambda spec, fresh=False: info) @@ -406,4 +406,4 @@ def test_a_reset_step_server_still_lists_its_tools(monkeypatch): assert "latex" in json.dumps(tools["step"]["inputSchema"]) assert tools == {t["name"]: t for t in mcp_bridge._tools(info)} # the page and the bridge never disagree monkeypatch.setattr(live, "probe", lambda spec, fresh=False: {**info, "mcp": [{"name": "look", "inputSchema": {"type": "object"}}]}) - assert "bridge_tools" not in client.get("/api/spaces/o/geo/live").json() + assert {t["name"] for t in client.get("/api/spaces/o/geo/live").json()["bridge_tools"]} == {"look", "reset", "state"} diff --git a/tests/test_model_relay.py b/tests/test_model_relay.py new file mode 100644 index 0000000000000000000000000000000000000000..4fce7dc338f82a70f831df202dd4dcd90d06602a --- /dev/null +++ b/tests/test_model_relay.py @@ -0,0 +1,23 @@ +from fastapi.testclient import TestClient + +from app import main, runner, runtime +from app.mimo.runner import core + + +def test_local_relay_precedes_capture_catchall_and_keeps_capability_auth(monkeypatch): + monkeypatch.setattr(runner, "_service", None) + client = TestClient(runner.service().capture.app) + monkeypatch.setattr(core, "by_cap", lambda cap: None) + assert client.get("/rlx/llm/invalid/health").status_code == 404 + assert client.post("/rlx/llm/invalid/v1/chat/completions", json={}).status_code == 404 + assert client.post("/v1/chat/completions", json={"model": "test", "messages": []}).status_code == 401 + async def relay(cap, request): + return {"route": "scoped-relay"} + monkeypatch.setattr(main, "llm_proxy", relay) + assert client.post("/rlx/llm/test/v1/chat/completions", json={}).json() == {"route": "scoped-relay"} + + +def test_local_and_deployed_capabilities_are_redacted_from_logs(): + for prefix in ("api", "rlx"): + msg = f"POST /{prefix}/llm/SECRET-CAP/v1/chat/completions" + assert "SECRET-CAP" not in runtime.SECRET_PATH.sub(lambda m: (m.group(1) or m.group(2)) + "[hidden]", msg) diff --git a/tests/test_nemo_runner.py b/tests/test_nemo_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..d2446393b7f4fd1fd92a9a57e7641cc5d1e43c19 --- /dev/null +++ b/tests/test_nemo_runner.py @@ -0,0 +1,73 @@ +import json + +import pytest + +from app import nemo_runner as runner +from app.envs import nemogym + + +def row(): + return {"responses_create_params": {"input": [{"role": "user", "content": "Look up A."}, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Previous reasoning"}]}], + "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}]}, + "expected_action": {"type": "function_call", "name": "lookup", "arguments": '{"id":"PRIVATE-ANSWER"}'}} + + +def test_responses_input_preserved_and_answer_withheld(): + data = row() + out = runner.responses_request(data) + assert out["input"] == data["responses_create_params"]["input"] + assert "PRIVATE-ANSWER" not in json.dumps(out) + + +def test_provider_hosted_tools_cannot_execute(): + data = row() + data["responses_create_params"]["tools"] = [{"type": "mcp", "server_url": "https://example.com", "require_approval": "never"}] + with pytest.raises(ValueError, match="Only function"): + runner.responses_request(data) + + +def test_native_comparator_scores_arguments_and_calls_before_text(): + data = row() + agent = nemogym.agent_of({}, "nvidia/Nemotron-RL-Agentic-Function-Calling-Pivot-v1") + expected = runner.validate(data, agent) + response = {"output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Looking it up"}]}, + {"type": "function_call", "name": "lookup", "arguments": '{"id":"PRIVATE-ANSWER"}'}]} + assert runner.score(expected, runner.prediction(response))[0] == 1 + response["output"][1]["arguments"] = '{"id":"wrong"}' + assert runner.score(expected, runner.prediction(response))[0] == 0 + response["output"][1]["arguments"] = 'not json' + assert runner.score(expected, runner.prediction(response))[0] == 0 + + +def test_invalid_reference_does_not_leak_through_errors(): + data = row() + data["expected_action"] = {"type": "PRIVATE-ANSWER"} + with pytest.raises(ValueError) as e: + runner.validate(data, nemogym.agent_of({}, "nvidia/Nemotron-RL-Agentic-Function-Calling-Pivot-v1")) + assert "PRIVATE-ANSWER" not in str(e.value) + + +def test_unknown_nemo_server_stays_native(): + with pytest.raises(ValueError, match="not hosted"): + runner.validate(row(), {"known": True, "resources_server": "another_server"}) + + +def test_provider_failure_inside_http_200_is_not_a_zero_reward(): + with pytest.raises(ValueError, match="did not produce a prediction"): + runner.prediction({"status": "failed", "error": {"message": "invalid schema"}, "output": []}) + with pytest.raises(ValueError, match="invalid Responses"): + runner.prediction({"message": "not a response"}) + + +def test_invalid_strict_declaration_is_relaxed_without_changing_parameters(): + data = row() + tool = data["responses_create_params"]["tools"][0] + tool["strict"] = True + original = json.dumps(data) + request = runner.responses_request(data) + assert request["tools"][0]["strict"] is False + assert request["tools"][0]["parameters"] == tool["parameters"] + assert json.dumps(data) == original + tool["parameters"].update(properties={}, required=[], additionalProperties=False) + assert runner.responses_request(data)["tools"][0]["strict"] is True diff --git a/tests/test_remote_mcp.py b/tests/test_remote_mcp.py new file mode 100644 index 0000000000000000000000000000000000000000..660947e3aa56fbd0f633aa03c6a85e9512d4ed13 --- /dev/null +++ b/tests/test_remote_mcp.py @@ -0,0 +1,62 @@ +"""Standard MCP sessions preserve cookies, negotiated headers and reply identity.""" +import json + +import httpx +import pytest + +from app import spaces_live as live + + +def test_standard_mcp_initializes_isolates_and_closes_sessions(): + seen, next_id = [], [] + def handle(request): + seen.append(request) + if request.method == "DELETE": + return httpx.Response(204) + message = json.loads(request.content) + if message["method"] == "initialize": + sid = str(len(next_id) + 1) + next_id.append(sid) + return httpx.Response(200, headers={"Mcp-Session-Id": sid, "Set-Cookie": f"episode={sid}; Path=/"}, + json={"jsonrpc": "2.0", "id": message["id"], "result": {"protocolVersion": "2025-06-18"}}) + sid = request.headers["mcp-session-id"] + assert request.headers["mcp-protocol-version"] == "2025-06-18" + assert request.headers["cookie"] == f"episode={sid}" + if message["method"] == "notifications/initialized": + assert "id" not in message + return httpx.Response(202) + # A progress notification is not the result; the actual event has multi-line data. + event = json.dumps({"jsonrpc": "2.0", "id": message["id"], "result": {"episode": sid}}, indent=2) + text = 'data: {"jsonrpc":"2.0","method":"notifications/progress"}\n\n' + '\n'.join('data: '+line for line in event.splitlines())+'\n\n' + return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, text=text) + with live.SpaceClient(transport=httpx.MockTransport(handle)) as a, live.SpaceClient(transport=httpx.MockTransport(handle)) as b: + first = live.RemoteMCP({"host": "https://fixture.hf.space"}, a) + second = live.RemoteMCP({"host": "https://fixture.hf.space"}, b) + assert first.call("tools/call", {}, 2) == {"episode": "1"} + assert second.call("tools/call", {}, 2) == {"episode": "2"} + assert first.call("tools/call", {}, 2) == {"episode": "1"} + first.close() + second.close() + assert len(next_id) == 2 + assert [r.headers["mcp-session-id"] for r in seen if r.method == "DELETE"] == ["1", "2"] + + +def test_expired_mcp_session_does_not_replay_or_reinitialize(): + calls = [] + def handle(request): + calls.append(request) + return httpx.Response(404) + with live.SpaceClient(transport=httpx.MockTransport(handle)) as c: + remote = live.RemoteMCP({"host": "https://fixture.hf.space"}, c) + remote.initialized, remote.session = True, "existing" + for _ in range(2): + with pytest.raises(live.SpaceError) as error: + remote.call("tools/call", {"name": "step"}, 2) + assert error.value.status == 410 + assert len(calls) == 1 + + +def test_mcp_rejects_an_unrelated_rpc_response(): + with live.SpaceClient(transport=httpx.MockTransport(lambda r: httpx.Response(200, json={"jsonrpc": "2.0", "id": "other", "result": {}}))) as c: + with pytest.raises(live.SpaceError, match="matching"): + live.RemoteMCP({"host": "https://fixture.hf.space"}, c).call("tools/list", {}, 2) diff --git a/tests/test_search_api.py b/tests/test_search_api.py index 29bf7d169067251b4a5b3dadd8d8a3eb2017306a..26a49328161dc88ccb362089416c5b24f79f1e9a 100644 --- a/tests/test_search_api.py +++ b/tests/test_search_api.py @@ -1,8 +1,7 @@ """/api/search and /api/search/tasks (app/search_api.py) over a catalog snapshot (no network: the listing is fixture rows). -What is checked: the Kind taxonomy the Explore page has always used (ORS with the other Spaces, FineEnvs' curated -Spaces as OpenEnv, Harbor whenever an index found tasks); pinned first in trending and FineEnvs' curated servers first -among OpenEnv Spaces; hidden environments left out of everything at query time; every answer (order, total, every +What is checked: declared framework taxonomy independent of editorial collections; metric-only trending with +separate featured picks; hidden environments left out of everything at query time; every answer (order, total, every facet's counts) equal to a plain-Python port of the page's own filtering over the same rows, for hundreds of queries (a parity test), through a published snapshot and through the live fallback alike; search text never read as FTS syntax; the visitor's own private datasets merged in; cache headers and ETags. @@ -124,9 +123,9 @@ def publish_fixture(tmp_path: Path) -> dict: def fw_js(d: dict) -> str: - """web/js/home.js `fw()`, verbatim, over catalog.environments()'s row (curated Spaces already OpenEnv there).""" + """No runtime evidence in the base fixture: metadata only supplies candidates.""" if d["kind"] == "space": - return "openenv" if d.get("framework") == "openenv" or d.get("openenv") else "other-space" + return "unverified-space" if d.get("framework") == "openenv" or d.get("openenv") else "other-space" return d.get("framework") or "harbor" @@ -143,16 +142,16 @@ def test_kind_taxonomy(tmp_path, mode): assert d["snapshot"]["source"] == mode expected: dict[str, int] = {} for r in visible(): - k = fw_js({**r, "framework": "openenv"} if r["key"] in COLLS[0]["ids"] and r["kind"] == "space" else r) + k = fw_js(r) expected[k] = expected.get(k, 0) + 1 assert {k: v for k, v in d["facets"]["kind"].items() if k != "all"} == expected assert d["facets"]["kind"]["all"] == d["total"] == len(visible()) - assert expected["other-space"] > 0 and expected["openenv"] > 0 + assert expected["other-space"] > 0 and expected["unverified-space"] > 0 cards = {c["key"]: c for c in d["rows"]} - # ORS sits with the other Spaces; FineEnvs' curated Spaces are OpenEnv whatever their manifest says + # Collections cannot supply framework evidence, including for FineEnvs. assert all(c["fw"] == "other-space" for c in cards.values() if c["kind"] == "space" and c["collection"] != "fineenvs" and c["framework"] == "ors") for key in ("space:fe/latex", "space:fe/ors-one", "space:fe/plain"): - assert cards[key]["fw"] == "openenv" and cards[key]["openenv"] and cards[key]["badges"] == ["OpenEnv"] + assert cards[key]["fw"] == "other-space" and not cards[key]["openenv"] assert cards["space:fe/plain"]["collection"] == "fineenvs" # the first collection naming it assert cards["ds/d08"]["fw"] == "harbor" # no framework at all: Harbor, as the page reads it for k in expected: @@ -161,18 +160,20 @@ def test_kind_taxonomy(tmp_path, mode): assert client.get("/api/search?k=ors").json()["query"]["kind"] == "other-space" # the page's old links -def test_pinned_first_and_curated_first(tmp_path): +def test_trending_uses_metrics_and_featured_is_separate(tmp_path): publish_fixture(tmp_path) rows = client.get("/api/search?size=100").json()["rows"] - assert {r["key"] for r in rows[:2]} == set(PINS) - assert all(r["pinned"] for r in rows[:2]) and not any(r["pinned"] for r in rows[2:]) - oe = client.get("/api/search?kind=openenv&size=100").json()["rows"] - assert oe[0]["key"] == "space:fe/latex" # pinned beats curated - curated = [r["collection"] == "fineenvs" for r in oe] - assert curated[:4] == [True] * 4 and not any(curated[4:]) # FineEnvs' servers lead the OpenEnv Spaces - assert client.get("/api/search?kind=openenv&sort=likes&size=100").json()["rows"][0]["likes"] == max(r["likes"] for r in oe) + want = sorted(visible(), key=lambda r: (-r["trending"], -r["likes"], -r["downloads"])) + assert [r["key"] for r in rows] == [r["key"] for r in want] + oe = client.get("/api/search?kind=unverified-space&size=100").json()["rows"] + assert [r["key"] for r in oe] == [r["key"] for r in want if fw_js(r) == "unverified-space"] + assert client.get("/api/search?kind=unverified-space&sort=likes&size=100").json()["rows"][0]["likes"] == max(r["likes"] for r in oe) trending = client.get("/api/search?trending=5&facets=0").json()["trending"] assert [r["key"] for r in trending] == [r["key"] for r in rows[:5]] + assert [r["key"] for r in client.get("/api/search?trending=5").json()["featured"]] == PINS + filtered = client.get("/api/search?kind=unverified-space&trending=48").json() + assert filtered["featured"] == [] + assert [r["key"] for r in filtered["trending"]] == [r["key"] for r in oe] def test_hidden_environments_are_left_out_everywhere(tmp_path): @@ -195,6 +196,55 @@ def test_an_admin_hiding_shows_without_a_new_snapshot(tmp_path, monkeypatch): assert client.get("/api/search?q=d01").json()["total"] == 0 # a settings change is a new question +def test_recent_openenv_checks_versions_and_trending_use_same_filters(tmp_path, monkeypatch): + from app import space_checks + import time + publish_fixture(tmp_path) + now = time.time() + evidence = {"fe/geo": {"schema": 1, "id": "fe/geo", "stage": "RUNNING", "checked_at": now, + "status": "API checked", "mode": "Simulation", "tools": 0, + "version": {"value": "0.2.3", "source": "Lockfile"}}} + monkeypatch.setattr(space_checks, "inventory", lambda: evidence) + params = {"kind": "openenv", "f": "health:API%20checked;oe:0.2.3", "trending": "24"} + data = client.get("/api/search", params=params).json() + assert data["total"] == 1 + assert [r["id"] for r in data["rows"]] == [r["id"] for r in data["trending"]] == ["fe/geo"] + assert data["rows"][0]["version_source"] == "Lockfile" + assert data["rows"][0]["tools_status"] == "No tools discovered" + assert data["facets"]["oe"] == [["0.2.3", 1]] + assert client.get("/api/search", params={**params, "f": "health:API%20checked;tools:Tools%20discovered"}).json()["total"] == 0 + # Negative checks and expiration invalidate cached search responses too. + evidence["fe/geo"]["status"] = "Checks failed" + assert client.get("/api/search", params=params).json()["total"] == 0 + evidence["fe/geo"].update(status="API checked", checked_at=now - space_checks.FRESH - 1) + assert client.get("/api/search", params=params).json()["total"] == 0 + expired = client.get("/api/search", params={"f": "health:Check%20expired"}).json() + assert expired["total"] == 1 and expired["rows"][0]["api_status"] == "Check expired" + + +def test_sql_python_parity_for_api_evidence(tmp_path, monkeypatch): + from app import space_checks + import time + publish_fixture(tmp_path) + now = time.time() + statuses = ["API checked", "Checks failed", "Not running", "Check unavailable"] + evidence = {r["id"]: {"id": r["id"], "status": statuses[i % 4], "checked_at": now - (i % 5) * 1000, + "mode": "Simulation", "tools": i % 3, "version": {"value": ">=0.2.3", "source": "Dependency constraint"}} + for i, r in enumerate(ROWS) if r.get("kind") == "space"} + monkeypatch.setattr(space_checks, "inventory", lambda: evidence) + ctx = search_api.context() + cards = search_api.py_cards([r for r in ROWS if not r.get("private")], ctx) + with snapshot.use() as (_, conn): + for facet, value in [("health", "API checked"), ("health", "Check expired"), ("health", "Not checked"), + ("mode", "Simulation"), ("tools", "Tools discovered"), ("oe", ">=0.2.3"), + ("oe_source", "Dependency constraint")]: + q = search_api.Query(size=100, sel={facet: [value]}) + actual, expected = search_api.sql_search(conn, q, ctx), search_api.py_search(cards, q) + assert actual["total"] == expected["total"] + assert actual["facets"] == expected["facets"] + assert [r["id"] for r in actual["rows"]] == [r["id"] for r in expected["matches"]] + + # ── parity with the page's own filtering ───────────────────────────────────── def random_queries(n: int, seed: int = 3) -> list[search_api.Query]: rnd = random.Random(seed) @@ -204,7 +254,8 @@ def random_queries(n: int, seed: int = 3) -> list[search_api.Query]: sel = {} for k, vals in (("type", ["Neither", "Benchmark", "OpenEnv", "ORS", "RL Environment"]), ("size", ["Under 100", "100 to 1,000", "1,000 or more", "Not indexed yet"]), - ("stage", ["Running", "Asleep or stopped", "Unknown"]), ("mcp", ["Has MCP tools", "No MCP tag"]), + ("stage", ["Running", "Sleeping", "Error", "Unknown"]), ("mcp", ["MCP tagged", "No MCP tag"]), + ("evidence", ["Manifest present", "Hub tag only"]), ("oe", ["0.2.1", "0.2.3"]), ("tags", TAGS)): if rnd.random() < 0.2: sel[k] = rnd.sample(vals, rnd.randint(1, 2)) @@ -305,7 +356,10 @@ def test_mine_merges_private_datasets(tmp_path, monkeypatch): assert {r["key"] for r in d["rows"]} == {"me/private-ds", "ds/d02"} and d["total"] == 2 assert d["facets"]["collection"]["mine"] == 2 and d["mine"] == 2 top = client.get("/api/search?mine=1&trending=3").json() - assert top["trending"][0]["key"] in PINS and "me/private-ds" in [r["key"] for r in top["trending"]] + assert top["trending"][0]["key"] == "me/private-ds" + assert [r["key"] for r in top["featured"]] == PINS + own = client.get("/api/search?mine=1&c=mine&trending=24").json() + assert {r["key"] for r in own["trending"]} == {"me/private-ds", "ds/d02"} assert top["total"] == len(visible()) + 1 and top["stats"]["datasets"] == sum(1 for r in visible() if r["kind"] == "dataset") + 1 assert next(r for r in top["rows"] if r["key"] == "me/private-ds")["private"] is True r = client.get("/api/search?mine=1") @@ -329,7 +383,8 @@ def test_responses_are_cards_only(tmp_path): assert len(d["rows"]) == 40 and len(d["trending"]) == 24 assert set(d["rows"][0]) == {"id", "key", "kind", "framework", "fw", "openenv", "collection", "heading", "brief", "downloads", "likes", "trending", "updated", "created", "stage", "mcp", "openenv_version", "manifest", "hardware", "badges", - "tags", "pinned", "mine", "indexed", "rollouts", "private"} + "tags", "pinned", "mine", "indexed", "rollouts", "private", "evidence", "stage_label", + "api_check", "api_status", "api_mode", "declared_version", "version_source", "tools_status"} assert [c["id"] for c in d["collections"]] == ["fineenvs", "terminal"] and "ids" not in d["collections"][0] assert client.get("/api/search?q=ds/d03").json()["rows"][0]["rollouts"] == 4 assert len(client.get("/api/search?trending=24").content) < 60_000 @@ -350,7 +405,7 @@ def test_environments_helper_for_whole_listing_readers(tmp_path, monkeypatch): rows = search_api.environments() assert [r["key"] for r in rows] == [r["key"] for r in ROWS if r["key"] not in HIDDEN and not r.get("private")] # listing order by = {r["key"]: r for r in rows} - assert by["space:fe/plain"]["framework"] == "openenv" and by["space:fe/plain"]["collection"] == "fineenvs" + assert by["space:fe/plain"]["framework"] == "space" and by["space:fe/plain"]["collection"] == "fineenvs" assert by["ds/d07"]["pinned"] and not by["ds/d07"]["gated"] and not by["ds/d07"]["private"] assert {r["key"] for r in search_api.environments(include_hidden=True)} >= HIDDEN @@ -383,11 +438,192 @@ def test_the_visitors_datasets_never_make_a_search_wait(monkeypatch): def test_a_spaces_rank_matches_the_trending_order_without_the_listing(tmp_path): publish_fixture(tmp_path) - key = lambda d: d["trending"] * 1e9 + d["likes"] * 1e4 + min(d.get("downloads") or 0, 9999) + key = lambda d: (-d["trending"], -d["likes"], -(d.get("downloads") or 0)) spaces = [r for r in visible() if r["kind"] == "space"] - order = sorted(spaces, key=lambda d: -key(d)) # stable: listing order breaks ties, as on the page + order = sorted(spaces, key=key) for i in (0, len(order) // 2, len(order) - 1): r = client.get(f"/api/search/rank?key={order[i]['key']}") assert r.status_code == 200 and r.json() == {"n": i + 1, "of": len(order)}, order[i]["key"] assert client.get(f"/api/search/rank?key={next(iter(HIDDEN))}").status_code == 404 # hidden: not ranked assert client.get("/api/search/rank?key=space:nobody/nothing").status_code == 404 + + +def test_runtime_and_evidence_filters_and_census_agree(tmp_path): + publish_fixture(tmp_path) + d = client.get("/api/search?kind=unverified-space&size=100").json() + assert sum(d["stats"]["openenv"].values()) == d["total"] + for evidence, count in d["stats"]["openenv"].items(): + r = client.get("/api/search", params={"kind": "unverified-space", "f": f"evidence:{evidence}", "trending": 48, "size": 100}).json() + assert r["total"] == count + assert all(x["evidence"] == evidence for x in r["rows"] + r["trending"]) + running = client.get("/api/search?kind=unverified-space&f=stage:Running").json() + assert running["total"] == d["stats"]["openenv_running"] + errors = client.get("/api/search?f=stage:Error&size=100").json() + assert errors["total"] and all(r["stage"] == "RUNTIME_ERROR" and r["stage_label"] == "Error" for r in errors["rows"]) + sleeping = client.get("/api/search?f=stage:Sleeping&size=100").json() + assert sleeping["total"] and all(r["stage"] == "SLEEPING" for r in sleeping["rows"]) + old = client.get("/api/search?f=stage:Asleep%20or%20stopped;mcp:Has%20MCP%20tools").json() + assert "Error" in old["query"]["f"]["stage"] and old["query"]["f"]["mcp"] == ["MCP tagged"] + + +def test_trending_priorities_cannot_overflow_into_each_other(tmp_path, monkeypatch): + # Old 1e9/1e4 weights let popularity displace a larger recent-interest score, + # and capped downloads made 10k and 10M downloads indistinguishable. + custom = [dict(ROWS[0], id=f"rank/{i}", key=f"rank/{i}", trending=t, likes=l, downloads=d) + for i, (t, l, d) in enumerate([(0, 10**9, 10**12), (0.1, 0, 0), (1, 3, 10000), (1, 3, 10**7)])] + monkeypatch.setattr(__import__(__name__, fromlist=["ROWS"]), "ROWS", custom) + monkeypatch.setattr(catalog, "pinned", lambda: ["rank/0"]) + publish_fixture(tmp_path) + got = client.get("/api/search?trending=4").json() + assert [r["id"] for r in got["rows"]] == ["rank/3", "rank/2", "rank/1", "rank/0"] + assert got["trending"] == got["rows"] + assert [r["id"] for r in got["featured"]] == ["rank/0"] + assert got["ranking"]["order"] == ["trending", "likes", "downloads"] + + +def test_empty_filters_have_no_unrelated_trending_or_featured(tmp_path): + publish_fixture(tmp_path) + d = client.get("/api/search?q=absent-environment-xyz&trending=24").json() + assert d["total"] == 0 and d["trending"] == [] and d["featured"] == [] + + +@pytest.mark.parametrize("mode", ["published", "fallback"]) +def test_owner_filter_is_exact_and_shared_by_results_facets_and_trending(tmp_path, monkeypatch, mode): + import sys + space = next(r for r in ROWS if r["kind"] == "space") + dataset = ROWS[0] + mixed = [*ROWS, {**space, "id": "ds/owned-space", "key": "space:ds/owned-space"}, + {**dataset, "id": "ds-copy/impostor", "key": "ds-copy/impostor", "heading": "ds environments"}] + monkeypatch.setattr(sys.modules[__name__], "ROWS", mixed) + if mode == "published": + publish_fixture(tmp_path) + expected = {r["key"] for r in visible() if r["id"].split("/")[0] == "ds"} + data = client.get("/api/search", params={"owner": "DS", "size": 100, "trending": 48}).json() + assert {r["key"] for r in data["rows"]} == expected + assert {r["kind"] for r in data["rows"]} == {"dataset", "space"} + assert data["total"] == data["facets"]["kind"]["all"] == len(expected) + assert {r["key"] for r in data["trending"]} == expected + assert all(r["id"].split("/")[0] == "ds" for r in data["featured"]) + query = search_api.Query(owner="DS") + ref = search_api.py_search(search_api.py_cards([r for r in mixed if not r.get("private")], search_api.context()), query) + assert {r["key"] for r in ref["matches"]} == expected + pages = [client.get("/api/search", params={"owner": "ds", "size": 10, "page": page, "facets": 0}).json()["rows"] + for page in range(1, (len(expected) + 9) // 10 + 1)] + assert {r["key"] for page in pages for r in page} == expected + assert sum(map(len, pages)) == len(expected) + filtered = client.get("/api/search", params={"owner": "ds", "kind": "openenv", "q": "owned-space"}).json() + assert all(r["id"] == "ds/owned-space" for r in filtered["rows"]) + for owner in ("d", "ds%", "ds/", "missing", "ds' OR 1=1 --"): + empty = client.get("/api/search", params={"owner": owner, "trending": 24}).json() + assert empty["total"] == 0 and empty["trending"] == [] and empty["featured"] == [] + + +def test_owner_filter_applies_to_authorized_private_dataset_merge(tmp_path, monkeypatch): + publish_fixture(tmp_path) + mine = [{**ROWS[0], "id": "me/private", "key": "me/private", "private": True}, + {**ROWS[0], "id": "someone-else/private", "key": "someone-else/private", "private": True}] + monkeypatch.setattr(search_api, "_mine_rows", lambda request: mine) + public = client.get("/api/search", params={"owner": "me"}).json() + assert public["total"] == 0 + response = client.get("/api/search", params={"owner": "ME", "mine": 1, "trending": 24}) + data = response.json() + assert data["total"] == 1 + assert data["stats"]["datasets"] == 1 + assert data["stats"]["dataset_tasks"] == (mine[0].get("indexed") or {}).get("tasks", 0) + assert [r["id"] for r in data["rows"]] == [r["id"] for r in data["trending"]] == ["me/private"] + assert data["rows"][0]["private"] is True + assert response.headers["cache-control"].startswith("private") + + +@pytest.mark.parametrize("status,stage,age,expected", [ + (None, "RUNNING", 0, "unverified-space"), + ("Checks failed", "RUNNING", 0, "unverified-space"), + ("API checked", "RUNNING", 0, "openenv"), + ("API checked", "RUNNING", 3601, "unverified-space"), + ("API checked", "SLEEPING", 0, "unverified-space"), +]) +def test_manifest_filename_never_establishes_openenv_without_fresh_live_evidence(tmp_path, monkeypatch, status, stage, age, expected): + from app import space_checks + publish_fixture(tmp_path) + target = next(r for r in ROWS if r.get("manifest")) + record = {"id": target["id"], "status": status, "stage": stage, "checked_at": time.time() - age} + monkeypatch.setattr(space_checks, "inventory", lambda: {target["id"]: record} if status else {}) + data = client.get("/api/search", params={"q": target["id"], "trending": 24}).json() + row = next(r for r in data["rows"] if r["id"] == target["id"]) + assert row["fw"] == expected + assert row["openenv"] is (expected == "openenv") + if expected != "openenv": + assert row["framework"] == "space" and "OpenEnv" not in row["badges"] + filtered = client.get("/api/search", params={"q": target["id"], "kind": "openenv", "trending": 24}).json() + assert bool(filtered["rows"]) is (expected == "openenv") + assert bool(filtered["trending"]) is (expected == "openenv") + assert filtered["facets"]["kind"].get(expected, 0) > 0 + + +def test_verified_untagged_space_has_protocol_facets(tmp_path, monkeypatch): + from app import space_checks + publish_fixture(tmp_path) + record = {"id": "fe/plain", "status": "API checked", "stage": "RUNNING", "checked_at": time.time(), "mode": "Simulation", "tools": 0} + monkeypatch.setattr(space_checks, "inventory", lambda: {"fe/plain": record}) + data = client.get("/api/search", params={"kind": "openenv", "f": "health:API%20checked"}).json() + assert [r["id"] for r in data["rows"]] == ["fe/plain"] + ctx = search_api.context() + cards = search_api.py_cards([r for r in ROWS if not r.get("private")], ctx) + with snapshot.use() as (_, conn): + q = search_api.Query(kind="openenv") + assert search_api.sql_search(conn, q, ctx)["facets"] == search_api.py_search(cards, q)["facets"] + + +def test_ready_scope_counts_and_facets_share_live_evidence(tmp_path, monkeypatch): + from app import space_checks + publish_fixture(tmp_path) + now = time.time() + good = {"id": "fe/geo", "status": "API checked", "stage": "RUNNING", "checked_at": now, + "task_catalog": {"checked_at": now, "tasks": 1234, "counted_splits": 1, "complete": True}} + checks = {"fe/geo": good, + "fe/latex": {**good, "id": "fe/latex", "checked_at": now - 3601, "task_catalog": {}}, + "fe/plain": {**good, "id": "fe/plain", "status": "Checks failed", "task_catalog": {}}, + "fe/ors-one": {**good, "id": "fe/ors-one", "status": "Checks failed", "interface": "ors", "task_catalog": {}}} + monkeypatch.setattr(space_checks, "inventory", lambda: checks) + data = client.get("/api/search", params={"scope": "ready", "owner": "fe", "size": 100, "trending": 24}).json() + assert {r["id"] for r in data["rows"]} == {"fe/geo", "fe/ors-one"} + assert data["stats"]["datasets"] == data["stats"]["dataset_tasks"] == 0 + assert data["stats"]["spaces"] == data["total"] == data["facets"]["kind"]["all"] == 2 + assert data["stats"]["space_tasks"] == 1234 + rank = client.get("/api/search/rank", params={"scope": "ready", "key": "space:fe/geo"}).json() + assert rank["of"] == 2 + assert client.get("/api/search/rank", params={"scope": "ready", "key": "space:fe/plain"}).status_code == 404 + ctx = search_api.context() + cards = search_api.py_cards([r for r in ROWS if not r.get("private")], ctx) + with snapshot.use() as (_, conn): + for q in (search_api.Query(scope="ready"), search_api.Query(scope="ready", owner="fe"), + search_api.Query(scope="ready", kind="openenv")): + sql = search_api.sql_search(conn, q, ctx, limit=100) + py = search_api.py_search(cards, q) + assert sql["total"] == py["total"] + assert sql["facets"] == py["facets"] + datasets = client.get("/api/search", params={"scope": "ready", "owner": "ds"}).json() + assert datasets["stats"]["datasets"] == datasets["total"] and datasets["stats"]["spaces"] == 0 + assert datasets["stats"]["dataset_tasks"] == len(TASKS["ds/d00"]) + empty = client.get("/api/search", params={"scope": "ready", "q": "nothing-matches"}).json() + assert all(empty["stats"][k] == 0 for k in ("datasets", "spaces", "dataset_tasks", "space_tasks")) + + +def test_fineenvs_promoted_without_pins_and_without_changing_trending(tmp_path, monkeypatch): + import sys + from app import space_checks + custom = [{**ROWS[0], "id": "FineEnvs/data", "key": "FineEnvs/data", "trending": 0}, + {**ROWS[1], "id": "other/popular", "key": "other/popular", "trending": 100}, + {**ROWS[-2], "id": "FineEnvs/server", "key": "space:FineEnvs/server", "kind": "space", "trending": 0}] + monkeypatch.setattr(sys.modules[__name__], "ROWS", custom) + monkeypatch.setattr(catalog, "pinned", list) + monkeypatch.setattr(space_checks, "inventory", lambda: {"FineEnvs/server": { + "id": "FineEnvs/server", "status": "API checked", "stage": "RUNNING", "checked_at": time.time()}}) + publish_fixture(tmp_path) + data = client.get("/api/search?scope=ready&trending=24").json() + assert [r["id"] for r in data["promoted"]] == ["FineEnvs/server", "FineEnvs/data"] + assert data["trending"][0]["id"] == "other/popular" + assert data["featured"] == [] + assert client.get("/api/search?scope=ready&owner=other&trending=24").json()["promoted"] == [] + assert client.get("/api/search?scope=ready&q=popular&trending=24").json()["promoted"] == [] + assert len(client.get("/api/search?scope=ready&owner=fineenvs&trending=24").json()["promoted"]) == 2 diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index 3d731706d25db38514ea4ff307d6b5e790c805d5..aa5b7ba2abd7ba9e6c738266d23bac11f7484e38 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -324,6 +324,30 @@ def test_indexer_run_publishes_and_prunes(tmp_path, monkeypatch): assert snapshot.check(store) and snapshot.current().counts["envs"] == 6 +def test_cached_listing_publication_preserves_discovery_timestamp(tmp_path): + at = time.time() - 1800 + indexer.run(snapshot.LocalStore(tmp_path / "store"), rows=ROWS, index=False, listing_at=at) + data = json.loads(gzip.decompress((config.STORAGE_DIR / "listing.json.gz").read_bytes())) + assert data["at"] == at + assert catalog._env_list["at"] == at + + +def test_timed_out_build_backs_off_so_later_batches_can_progress(monkeypatch): + class Stalled: + def __init__(self, **kwargs): pass + def start(self): pass + def join(self, **kwargs): pass + def is_alive(self): return True + monkeypatch.setattr(indexer.threading, "Thread", Stalled) + items = [{"spec": "snaptest/large", "reason": "new", "tagged": True}] + state = {"failed": {}, "not_harbor": {}, "rebuild": []} + info = lambda spec: {"sha": "same-revision"} + first = indexer.refresh_indexes(items, state, budget_s=90, max_builds=4, info=info) + assert first["abandoned"] == ["snaptest/large"] + second = indexer.refresh_indexes(items, state, budget_s=90, max_builds=4, info=info) + assert second["abandoned"] == [] and second["skipped"][0]["spec"] == "snaptest/large" + + def test_a_failed_run_publishes_nothing(tmp_path, monkeypatch): store = store_of(tmp_path) first = indexer.run(store, rows=ROWS, index=False)["pointer"] diff --git a/tests/test_space_checks.py b/tests/test_space_checks.py new file mode 100644 index 0000000000000000000000000000000000000000..4e3abd49b0849ba45510f6428d5b8db8f7baeb74 --- /dev/null +++ b/tests/test_space_checks.py @@ -0,0 +1,135 @@ +import json +import time + +import pytest + +from app import space_checks as checks, spaces_live as live + + +def facts(): + api = {"info": {"version": "1.0.0"}, "paths": { + "/reset": {"post": {}}, "/step": {"post": {}}, "/state": {"get": {}}, "/mcp": {"post": {}}}} + return [api, {"status": "healthy"}, {"name": "test", "description": "test", "version": "9.9.9"}, + {"action": {}, "observation": {}, "state": {}}, None] + + +def test_typed_actions_do_not_require_mcp_but_production_does(): + f = facts() + assert checks.assess(*f)["status"] == checks.PASS + f[0]["paths"] = {"/mcp": {"post": {}}} + assert checks.assess(*f)["status"] == "Checks failed" + f[-1] = [{"name": "echo"}] + assert checks.assess(*f)["mode"] == "MCP service" + + +@pytest.mark.parametrize("change", ["schema", "health", "route", "method", "html"]) +def test_failures_cannot_become_api_checked(change): + f = facts() + if change == "schema": + f[3].pop("state") + elif change == "health": + f[1]["status"] = "ok" + elif change == "route": + f[0]["paths"].pop("/step") + elif change == "method": + f[0]["paths"]["/step"] = {"get": {}} + else: + f[0] = "Space UI" + assert checks.assess(*f)["status"] == "Checks failed" + + +@pytest.mark.parametrize("dependency,value,source", [ + ("openenv-core==0.2.3", "0.2.3", "Dependency pin"), + ("openenv==0.7.0", "0.7.0", "Dependency pin"), + ("openenv_core[core]>=0.2.0,<0.3", "<0.3,>=0.2.0", "Dependency constraint"), + ("openenv-core==0.2.*", "==0.2.*", "Dependency constraint"), + ("openenv-core", "Unpinned", "Dependency constraint"), + ("openenv-core @ git+https://github.com/meta-pytorch/OpenEnv.git@abc123", "git:abc123", "Source reference"), + ('openenv-core==0.2.3; python_version < "3.12"', "Conditional", "Dependency constraint"), +]) +def test_declared_versions_keep_constraints_and_source_refs(dependency, value, source): + v = checks.requirement_version(dependency) + assert (v["value"], v["source"]) == (value, source) + + +def test_version_does_not_use_environment_or_api_version(): + assert checks.parse_version('[project]\nname="env"\nversion="9.9.9"\ndependencies=["fastapi"]', "pyproject.toml") is None + assert checks.requirement_version("gymnasium==1.0.0") is None + assert checks.tag_version(["openenv", "openenv-nonsense"])["value"] == "Unknown" + assert checks.tag_version(["openenv-0.2.3"])["value"] == "0.2.3" + + +def test_lock_and_uv_source_precedence(): + doc = '[[package]]\nname="openenv-core"\nversion="0.7.1.dev0"\nsource={git="https://example.org/env?rev=main#abcdef"}' + assert checks.parse_version(doc, "uv.lock") == {"value": "git:abcdef", "source": "Lockfile"} + doc = '[project]\ndependencies=["openenv-core>=0.2"]\n[tool.uv.sources]\nopenenv-core={git="https://example.org/env",rev="v0.2.3"}' + assert checks.parse_version(doc, "pyproject.toml")["value"] == "git:v0.2.3" + assert checks.parse_version(doc.replace("openenv-core", "openenv"), "pyproject.toml")["value"] == "git:v0.2.3" + + +def test_expiry_future_dates_and_newer_failures(tmp_path, monkeypatch): + monkeypatch.setattr(checks.config, "STORAGE_DIR", tmp_path) + now = time.time() + passing = {"schema": 1, "id": "owner/env", "checked_at": now - 10, "status": checks.PASS} + checks.save(passing) + assert checks.status(checks.inventory()["owner/env"], now) == checks.PASS + assert checks.status(passing, now + checks.FRESH) == "Check expired" + assert checks.status(passing, now - 20) == "Check expired" + checks.save({**passing, "checked_at": now, "status": "Checks failed"}) + checks.save(passing) # late completion cannot overwrite a newer failure + assert checks.status(checks.inventory()["owner/env"], now) == "Checks failed" + + +def test_sleeping_space_is_not_contacted_or_woken(tmp_path, monkeypatch): + monkeypatch.setattr(checks.config, "STORAGE_DIR", tmp_path) + monkeypatch.setattr(live, "record", lambda *a, **kw: {"stage": "SLEEPING"}) + monkeypatch.setattr(live, "SpaceClient", lambda *a, **kw: pytest.fail("contacted sleeping server")) + r = checks.check("owner/sleeping") + assert r["status"] == "Not running" + + +def test_network_failure_invalidates_previous_pass(tmp_path, monkeypatch): + monkeypatch.setattr(checks.config, "STORAGE_DIR", tmp_path) + checks.save({"schema": 1, "id": "owner/env", "checked_at": time.time() - 10, "status": checks.PASS}) + def offline(*a, **kw): + raise ConnectionError("offline") + monkeypatch.setattr(live, "record", offline) + assert checks.check("owner/env")["status"] == "Check unavailable" + assert checks.status(checks.inventory()["owner/env"]) != checks.PASS + + +def test_concurrent_refresh_skips_locked_queue(tmp_path, monkeypatch): + fcntl = pytest.importorskip("fcntl") + monkeypatch.setattr(checks.config, "CACHE_DIR", tmp_path) + monkeypatch.setattr(checks, "_scan", lambda: pytest.fail("started duplicate queue")) + with (tmp_path / "openenv-checks.lock").open("a") as lease: + fcntl.flock(lease, fcntl.LOCK_EX | fcntl.LOCK_NB) + assert checks.scan() == [] + + +def test_automatic_queue_includes_inactive_and_untagged_without_starvation(): + candidates = [(f"org/active{i}", "RUNNING") for i in range(120)] + [(f"org/sleep{i}", "SLEEPING") for i in range(40)] + candidates += [("FineEnvs/recent", "RUNNING"), ("FineEnvs/new", "RUNNING")] + now = time.time() + old = {"FineEnvs/recent": {"stage": "RUNNING", "checked_at": now, "status": checks.PASS, "task_catalog": {}}} + pending = checks.pending(candidates, old, now) + assert len(pending) == 128 and pending[0] == "FineEnvs/new" + assert sum("sleep" in spec for spec in pending) == 32 + assert "FineEnvs/recent" not in pending + for spec in pending: + old[spec] = {"checked_at": now, "stage": "SLEEPING", "status": "Not running"} + next_batch = checks.pending(candidates, old, now) + assert next_batch and not set(next_batch) & set(pending) + + +def test_other_environment_interfaces_need_published_methods_and_fresh_running_checks(): + api = {"info": {"version": "1"}, "paths": {"/seed_session": {"post": {}}, "/verify": {"post": {}}}} + assert checks.environment_interface(api) == "nemo-gym" + rec = {"interface": "nemo-gym", "status": "Checks failed", "stage": "RUNNING", "checked_at": time.time()} + assert checks.browseable(rec) and not checks.verified(rec) + assert not checks.browseable({**rec, "checked_at": time.time() - checks.FRESH}) + assert not checks.browseable({**rec, "stage": "SLEEPING"}) + assert not checks.browseable({**rec, "status": "Check unavailable"}) + api["paths"]["/verify"] = {"get": {}} + assert checks.environment_interface(api) is None + assert checks.environment_interface(facts()[0]) is None # generic reset/step alone is insufficient diff --git a/tests/test_space_tasks.py b/tests/test_space_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..1d30a2874668ef576fea39ee3bed24ea46990212 --- /dev/null +++ b/tests/test_space_tasks.py @@ -0,0 +1,41 @@ +import time + +import pytest + +from app import space_tasks as tasks, spaces_live as live + + +def test_multiple_environments_count_once_and_keep_unknowns(monkeypatch): + docs = {"/list_environments": ["a", "a", "b", "../bad"], + "/a/splits": [{"name": "train", "num_tasks": 12}, {"name": "train", "num_tasks": 12}], + "/b/splits": [{"name": "test", "num_tasks": True}, {"name": "valid", "num_tasks": 0}]} + monkeypatch.setattr(live, "_get_json", lambda rec, path: docs[path]) + d = tasks.discover({}, {"/{env_name}/splits"}) + assert [e["env"] for e in d["environments"]] == ["a", "b"] + s = tasks.summary(d) + assert s["tasks"] == 12 and s["counted_splits"] == 2 and s["unknown_splits"] == 1 + assert not s["complete"] + + +@pytest.mark.parametrize("value", [True, -1, "100", 2.5, float("inf"), 10**30]) +def test_invalid_counts_never_inflate_totals(value): + assert tasks.count(value) is None + + +def test_census_filters_visibility_expiry_and_inactive_spaces(): + now = time.time() + s = {"tasks": 19, "counted_splits": 1, "checked_at": now, "complete": True} + records = {"a": {"stage": "RUNNING", "task_catalog": s}, + "private": {"stage": "RUNNING", "task_catalog": s}, + "asleep": {"stage": "SLEEPING", "task_catalog": s}, + "stale": {"stage": "RUNNING", "task_catalog": {**s, "checked_at": now - tasks.FRESH}}} + assert tasks.census(records, ["a", "asleep", "stale"], now)["tasks"] == 19 + assert tasks.census(records, ["a"], now - 1)["tasks"] == 0 + + +def test_task_environment_must_be_advertised(monkeypatch): + monkeypatch.setattr(live, "probe", lambda spec: {"running": True, "task_api": {"env": "a", "environments": [{"env": "a"}, {"env": "b"}]}}) + monkeypatch.setattr(live, "record", lambda spec: {}) + assert live._task_api("o/s", "b")[1] == "b" + with pytest.raises(live.SpaceError): + live._task_api("o/s", "../secrets") diff --git a/tests/ui-audit.mjs b/tests/ui-audit.mjs index b92470065bac190ae3b3ffe9c8edcb80a9b5c126..e226b1991f72baee63e17f2e4d1a9d270d660e41 100644 --- a/tests/ui-audit.mjs +++ b/tests/ui-audit.mjs @@ -205,8 +205,8 @@ async function health(p, label) { await go(p, "#/task/music-gK-0768", 600); ok("MiMo explorer's addresses still open", await until(p, ".tp-sec", 40000) && (await here(p)) === `/t/${MIMO}/music-gK-0768`); await go(p, "#/community?view=tasks", 600); - ok("community: the tasks nobody has tried", await until(p, "#ct-rows .runrow", 60000) && /not tried yet/.test(await p.textContent("#cm-top"))); - await health(p, "community tasks 1440"); + ok("community: legacy Tasks links open the rollout feed", await until(p, "#cm-filters select[data-f=runner]", 60000) && !(await p.$("#cm-view"))); + await health(p, "community rollouts 1440"); // a live Space: playground, tasks, connect await go(p, "#/s/FineEnvs/geoguesser-env", 600); @@ -218,7 +218,7 @@ async function health(p, label) { await p.click("#pg-reset"); ok("space: a reset starts an episode with an image", await until(p, "#pg-log .rv-img", 40000)); await p.selectOption("#pg-tool", "look"); - await p.fill("#pgt-heading_deg", "90"); + await p.getByRole("spinbutton", { name: "heading_deg", exact: true }).fill("90"); await p.click("#pg-call"); await p.waitForTimeout(5000); ok("space: a tool call answers in the episode", /Facing 90/.test(await p.textContent("#pg-log"))); @@ -240,7 +240,7 @@ async function health(p, label) { ok("space (reset/step): fills in", await liveIn(p)); await p.click("#pg-reset"); ok("space (reset/step): reset shows the image", await until(p, "#pg-log .rv-img", 40000)); - await p.fill("#pga-latex", "x^2"); + await p.getByRole("textbox", { name: /^latex$/i }).fill("x^2"); await p.click("#pg-step"); ok("space (reset/step): a step earns a reward", await until(p, "#pg-log .pg-entry:last-of-type .rv-reward", 40000)); await health(p, "space latex 1440"); @@ -253,8 +253,8 @@ async function health(p, label) { await p.click("#pg-send"); await p.waitForTimeout(2500); await p.selectOption("#pg-route", { label: "POST /guess" }); - ok("nemo gym: a short string is a one-line input", await p.$("input#pgr-word")); - await p.fill("#pgr-word", "crane"); + ok("nemo gym: a short string is a one-line input", await p.$('input[data-k="word"]')); + await p.getByRole("textbox", { name: "Word", exact: true }).fill("crane"); await p.click("#pg-send"); await p.waitForTimeout(2500); await p.selectOption("#pg-route", { label: "POST /get_history" }); diff --git a/uv.lock b/uv.lock index b4927974fd7a452b35efcb36e456ae579839adc5..fc828c7c04152898f08327c53638faf7ea0426cc 100644 --- a/uv.lock +++ b/uv.lock @@ -3099,6 +3099,7 @@ dependencies = [ { name = "langdetect" }, { name = "openenv" }, { name = "openpyxl" }, + { name = "packaging" }, { name = "pandas" }, { name = "pillow" }, { name = "pyarrow" }, @@ -3124,6 +3125,7 @@ requires-dist = [ { name = "langdetect", specifier = ">=1.0.9" }, { name = "openenv", git = "https://github.com/huggingface/OpenEnv?rev=49aa302ba5c67041d11da128139ba3ae15529543" }, { name = "openpyxl", specifier = ">=3.1" }, + { name = "packaging", specifier = ">=24.2" }, { name = "pandas", specifier = ">=2.2" }, { name = "pillow", specifier = ">=10" }, { name = "pyarrow", specifier = ">=17" }, diff --git a/web-admin/admin.css b/web-admin/admin.css index 89de417e460d83f17cd7fa060efde0b77c9904fa..4126d04f4787072ffb830b640a12d3a6e3d64c73 100644 --- a/web-admin/admin.css +++ b/web-admin/admin.css @@ -139,3 +139,35 @@ .ad-nav a { flex: none; } .ad-nav a.on { box-shadow: inset 0 -2px 0 var(--text); border-radius: 8px 8px 0 0; } } @media (max-width: 760px) { .ad-hello .ad-actions { margin-left: 0; } .ad-pstats { grid-template-columns: repeat(2, minmax(0, 1fr)); } .ad-audit li { grid-template-columns: minmax(0, 1fr); } } + +/* Evidence and maintenance are separate from dataset indexes and saved runs. */ +.ad-operations { margin: 16px 0; } +.ad-ops-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 18px 24px; } +.ad-ops-grid > div { min-width: 0; } +.ad-ops-grid p { margin: 6px 0 10px; overflow-wrap: anywhere; } +.ad-ops-grid dd, .ad-ops-grid dt { min-width: 0; overflow-wrap: anywhere; } +.ad-check-states, .ad-runner-summary { display: flex; flex-wrap: wrap; gap: 6px; } +.ad-check-states, .ad-ops-note { grid-column: 1 / -1; } +.ad-check-states .chip { max-width: 100%; white-space: normal; overflow-wrap: anywhere; } +.ad-ops-note summary { cursor: pointer; margin-bottom: 10px; } +.ad-tools .input, .ad-tools select { min-width: 0; max-width: 100%; width: auto; } +.ad-tools .search { min-width: 0; } +.ad-name > .chip { max-width: 100%; overflow: hidden; text-overflow: ellipsis; } +.ad-stat b { overflow-wrap: anywhere; } +.ad-task, .ad-model, .ad-when, .ad-act { min-width: 0; } +.ad-main .panel { min-width: 0; } +.ad-main .tbl { overflow-x: auto; max-width: 100%; } +.ad-main #ad-run-error { margin-bottom: 10px; color: var(--muted); font-size: 13px; } +@media (max-width: 760px) { + .ad-ops-grid { grid-template-columns: minmax(0, 1fr); } + .ad-hello h1 { overflow-wrap: anywhere; } + .ad-stat { padding: 10px; gap: 8px; } + .ad-stat .ic-wrap { display: none; } + .ad-tools .search { flex-basis: 100%; } + .ad-tools > select { flex: 1 1 130px; } + .ad-head .ad-actions { width: 100%; } + .ad-run { gap: 8px; } + .ad-run .ad-act { grid-column: 1 / -1; } +} + +@media (min-width: 1101px) and (max-width: 1450px) { .ad-stats { grid-template-columns: repeat(3, minmax(0, 1fr)); } } diff --git a/web-admin/admin.js b/web-admin/admin.js index ff5fd3a65b57bfc88eb24327410fc82448201074..2065c8bb65f14d7b689c830d369bf3c10f0711e7 100644 --- a/web-admin/admin.js +++ b/web-admin/admin.js @@ -6,7 +6,7 @@ import { icon } from "/js/icons.js"; import { getSession } from "/js/session.js"; const TABS = [["overview", "Overview", "grid"], ["rollouts", "Rollouts", "play"], ["people", "People", "users"], ["environments", "Environments", "database"], - ["collections", "Collections", "layout"], ["indexes", "Indexes", "archive"], ["settings", "Settings", "gauge"], ["audit", "Audit log", "list"]]; + ["collections", "Collections", "layout"], ["indexes", "Indexing & API checks", "archive"], ["settings", "Settings", "gauge"], ["audit", "Audit log", "list"]]; const enc = (s) => s.split("/").map(encodeURIComponent).join("/"); const compact = new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }); const bytes = (n) => (n == null ? "–" : n > 1e6 ? `${(n / 1e6).toFixed(1)} MB` : `${Math.round(n / 1e3)} KB`); @@ -14,8 +14,9 @@ let EXPLORER = ""; // the public explorer: datasets, Spaces and rollouts open const envHref = (e) => `${EXPLORER}/${e.kind === "space" ? "s" : "d"}/${enc(e.id)}`; const runHref = (id) => `${EXPLORER}/run/${encodeURIComponent(id)}`; const thumb = (id) => `https://cdn-thumbnails.huggingface.co/social-thumbnails/spaces/${enc(id)}.png`; -let alive = true, timer = null, counts = {}; -export function unmount() { alive = false; clearTimeout(timer); } +let alive = true, timer = null, counts = {}, events = null; +const runnerName = v => ({harbor:"Harbor", mimo:"MiMo", "nemo-gym":"NeMo Gym", openenv:"OpenEnv", verifiers:"Verifiers", verl:"verl", skyrl:"SkyRL"}[v] || v || "Unknown"); +export function unmount() { alive = false; clearTimeout(timer); events?.abort(); } // a person: their Hub picture (or their initial), their name, a link to their profile const pfp = (name, src, size = 22) => (src ? `` @@ -26,6 +27,7 @@ const person = (name, src, { link = true, filter = false } = {}) => `
${sk.lines(80, 60, 90, 70)}
`; ({ overview, rollouts, people, environments, collections, indexes, settings, audit })[tab](body, { go, preset, s }).catch((e) => { - if (alive) body.innerHTML = `
${icon("alert")}${esc(e.message)}
`; + if (alive && body.isConnected) body.innerHTML = `
${icon("alert")}${esc(e.message)}
`; }); window.scrollTo(0, 0); }; @@ -59,7 +62,7 @@ export async function mount(el, { qs, explorer }) { if (t) { e.preventDefault(); preset.user = ""; go(t.dataset.tab); } const u = e.target.closest("[data-user]"); if (u) { e.preventDefault(); go("rollouts", { user: u.dataset.user }); } - }); + }, {signal:events.signal}); show(); // live counts beside the sections: running rollouts, new environments api("/api/admin/overview").then((o) => { counts = { rollouts: o.rollouts.live || "", environments: o.new.length || "" }; if (alive) nav(); }).catch(() => {}); @@ -69,15 +72,16 @@ const head = (title, sub, actions = "") => `

${titl const stat = (ic, n, label, sub = "", tone = "") => `
${icon(ic, 16)}
${n}${label}${sub ? `${sub}` : ""}
`; const runRow = (r, actions = false) => `
${statusPill(r.status)} - ${esc(r.title)}${esc(r.dataset)} + ${esc(r.title || r.path)}${esc(r.dataset)} · ${esc(runnerName(r.runner))}${r.community ? "On Community" : r.moderated ? "Moderated" : r.visibility === "public" ? "Public · not eligible for Community" : "Private"} ${person(r.user, r.avatar, { filter: true })} - ${esc((r.model || "").split("/")[1] || r.model || "endpoint")}${esc(r.harness)} + ${esc((r.model || "").split("/")[1] || r.model || "endpoint")}${esc(r.harness || runnerName(r.runner))} ${rewardBadge(r.reward, r.status)} ${ago(r.created_at)}${dur(r.wall_s || ((r.finished_at || Date.now() / 1000) - (r.started_at || r.created_at)))} ${actions ? `${r.live ? `` : ""} - ${r.moderated ? `` : r.visibility === "public" ? `` : ""}` : ""}
`; + ${r.moderated ? `` : r.community ? `` : ""}` : ""}

`; async function runActions(body, reload) { + body.runEvents?.abort(); body.runEvents = new AbortController(); body.addEventListener("click", async (e) => { const c = e.target.closest("[data-cancel]"), h = e.target.closest("[data-hide]"), u = e.target.closest("[data-unhide]"); try { @@ -85,45 +89,47 @@ async function runActions(body, reload) { await api(`/api/admin/rollouts/${c.dataset.cancel}/cancel`, { method: "POST" }); toast("Asked the explorer to stop it: a few seconds"); } else if (h) { if (!confirm("Take this rollout off Community? Its owner still sees it.")) return; await api(`/api/admin/rollouts/${h.dataset.hide}/moderate`, { method: "POST", body: { hidden: true } }); toast("Hidden from Community"); } - else if (u) { await api(`/api/admin/rollouts/${u.dataset.unhide}/moderate`, { method: "POST", body: { hidden: false } }); toast("Back on Community"); } + else if (u) { await api(`/api/admin/rollouts/${u.dataset.unhide}/moderate`, { method: "POST", body: { hidden: false } }); toast("Moderation removed; Community eligibility still applies"); } else return; reload(); } catch (err) { toast(err.message); } - }); + }, {signal:body.runEvents.signal}); } // ── overview ───────────────────────────────────────────────────────────────── async function overview(body, { go, s }) { const o = await api("/api/admin/overview"); - if (!alive) return; + if (!alive || !body.isConnected) return; const r = o.rollouts, e = o.environments, on = o.settings.rollouts_enabled; const peak = Math.max(1, ...r.by_day.map(([, n]) => n)); - const live = o.recent.filter((x) => x.live); + const live = o.live; const last = o.recent[0]; body.innerHTML = `
${pfp(s.user.name, s.user.avatar, 44)}

Hi, ${esc(s.user.name)}

Rollouts are ${on ? "on" : "paused"} · ${fmt.format(r.live)} running now${last ? ` · last one ${ago(last.created_at)}` : ""}

-
+
- ${stat("play", fmt.format(r.live), "running now", `${fmt.format(r.today)} started today`, r.live ? "live" : "")} - ${stat("list", fmt.format(r.total), "rollouts in all", `${fmt.format(r.week)} this week`)} - ${stat("users", fmt.format(r.users), "people", "who ran one")} + ${stat("play", fmt.format(r.live), "running now", `${fmt.format(r.today)} in the last 24 hours`, r.live ? "live" : "")} + ${stat("list", fmt.format(r.total), "rollouts in all", `${fmt.format(r.week)} in the last 7 days`)} + ${stat("users", fmt.format(r.public), "on Community", `${fmt.format(r.private)} private · ${fmt.format(r.users)} people`)} ${stat("alert", r.finished ? `${Math.round((100 * r.failed) / r.finished)}%` : "–", "failed or stopped", `of ${fmt.format(r.finished)} finished`, r.finished && r.failed / r.finished > 0.3 ? "warn" : "")} - ${stat("database", fmt.format(e.datasets), "datasets", `${fmt.format(e.indexed)} Harbor indexed`)} - ${stat("globe", fmt.format(e.spaces), "environment Spaces", `${fmt.format(e.openenv)} OpenEnv`)} + ${stat("database", fmt.format(e.public.datasets), "visible datasets", `${fmt.format(e.public.tasks)} indexed task records`)} + ${stat("globe", fmt.format(e.public.spaces), "visible checked Spaces", `${fmt.format(e.spaces)} discovery candidates`)}
+ ${operations(o.operations)} ${live.length ? `

Running now

All rollouts
${live.map((x) => runRow(x, true)).join("")}
` : ""} +
${r.by_runner.map(([name,n]) => `${esc(runnerName(name))} · ${fmt.format(n)}`).join("")}
-

${icon("play", 15)}Rollouts, last 14 days

${fmt.format(r.week)} this week
+

${icon("play", 15)}Rollouts, last 14 days

${fmt.format(r.week)} in the last 7 days
${r.by_day.map(([d, n]) => `
${d.slice(8)}
`).join("")}
${r.by_model.length ? `
Most-used models
    ${r.by_model.map(([m, n]) => `
  • ${esc(m)} ${fmt.format(n)}
    • `).join("")}
` : ""}

${icon("flame", 15)}New on the Hub

Curate
${o.new.length ? `` + ${esc(x.id)}${x.kind === "space" ? "Space" : "Dataset"} · ${ago(Date.parse(x.created) / 1000)}${x.collection ? ` · in ${esc(x.collection)}` : ""}`).join("")}` : `

Nothing new in the last two weeks.

`}

${icon("clock", 15)}Latest rollouts

All rollouts
@@ -140,40 +146,49 @@ const pick = (st) => ({ rollouts_enabled: st.rollouts_enabled, max_active: st.ma // ── rollouts ───────────────────────────────────────────────────────────────── async function rollouts(body, { preset }) { - const st = { status: "", q: "", user: preset.user || "" }; - body.innerHTML = `${head("Rollouts", "Everyone's, private ones too, with who ran them. Click a person to see only theirs.")} -
${icon("search", 16)}
-
${[["", "All"], ["live", "Running"], ["done", "Done"], ["failed", "Failed"], ["cancelled", "Stopped"], ["interrupted", "Interrupted"]].map(([k, l]) => ``).join("")}
-
-
${sk.lines(80, 70, 90)}
`; + const st = { status: "", q: "", user: preset.user || "", runner: "", visibility: "", limit: 100 }; + let sequence = 0, pending = null; + body.innerHTML = `${head("Rollouts", "All saved runners, including private runs. Community eligibility requires a completed, graded, unrestricted public rollout.")} +
${icon("search", 16)}
+ + + +
+
${sk.lines(80, 70, 90)}
+
`; const load = async () => { - const d = await api(`/api/admin/rollouts?status=${st.status}&q=${encodeURIComponent(st.q)}&user=${encodeURIComponent(st.user)}&limit=200`); - if (!alive) return; - $("#ad-who", body).innerHTML = st.user ? `` : ""; - $("#ad-runs", body).innerHTML = d.runs.length ? `

${fmt.format(d.total)} rollout${d.total === 1 ? "" : "s"}

${d.runs.map((r) => runRow(r, true)).join("")}` - : emptyState("filter", "No rollouts here", "Try another filter."); - clearTimeout(timer); - if (d.runs.some((r) => r.live)) timer = setTimeout(load, 4000); + clearTimeout(timer); pending?.abort(); pending = new AbortController(); const request = ++sequence; + try { + const d = await api(`/api/admin/rollouts?${new URLSearchParams(st)}`, {signal:pending.signal}); + if (!alive || !body.isConnected || request !== sequence) return; + $("#ad-run-error",body).textContent = ""; + $("#ad-runner",body).innerHTML = '' + d.runners.map(v=>``).join(""); + $("#ad-who", body).innerHTML = st.user ? `` : ""; + $("#ad-runs", body).innerHTML = d.runs.length ? `

Showing ${fmt.format(d.runs.length)} of ${fmt.format(d.total)} rollouts

${d.runs.map((r) => runRow(r, true)).join("")}` : emptyState("filter", "No rollouts here", "Try another filter."); + $("#ad-run-more",body).hidden = d.runs.length >= d.total || st.limit >= 500; + if (d.total > 500 && st.limit >= 500) $("#ad-run-error",body).textContent = "Showing the latest 500 matches. Refine your filters to find older runs."; + if (d.runs.some(r=>r.live)) timer = setTimeout(()=>body.isConnected && load(), 5000); + } catch(e) { if(e.name!=="AbortError" && body.isConnected) $("#ad-run-error",body).textContent=e.message; } }; let t; - $("#ad-q", body).addEventListener("input", (e) => { clearTimeout(t); t = setTimeout(() => { st.q = e.target.value.trim(); load(); }, 200); }); - $("#ad-st", body).addEventListener("click", (e) => { const b = e.target.closest("[data-st]"); if (!b) return; st.status = b.dataset.st; - body.querySelectorAll("#ad-st [data-st]").forEach((x) => x.setAttribute("aria-pressed", String(x === b))); load(); }); - body.addEventListener("click", (e) => { if (e.target.closest("#ad-clear-user")) { st.user = ""; preset.user = ""; load(); } }); - runActions(body, load); - await load(); + $("#ad-q", body).addEventListener("input", e=>{clearTimeout(t); pending?.abort(); ++sequence; st.q=e.target.value.trim(); st.limit=100; t=setTimeout(()=>body.isConnected && load(),250);}); + for(const [id,key] of [["ad-runner","runner"],["ad-vis","visibility"],["ad-st","status"]]) $("#"+id,body).addEventListener("change",e=>{st[key]=e.target.value;st.limit=100;load();}); + $("#ad-refresh",body).addEventListener("click",load); + $("#ad-run-more",body).addEventListener("click",()=>{st.limit+=100;load();}); + body.addEventListener("click",e=>{if(e.target.closest("#ad-clear-user")){st.user="";preset.user="";load();}}); + runActions(body,load); await load(); } // ── people ─────────────────────────────────────────────────────────────────── async function people(body, { go }) { const d = await api("/api/admin/people"); - if (!alive) return; + if (!alive || !body.isConnected) return; body.innerHTML = `${head("People", "Everyone who has run a rollout, most recent first.")} ${d.people.length ? `
${d.people.map((p) => `
${pfp(p.name, p.avatar, 40)}
${esc(p.name)} ${icon("external", 12)} active ${ago(p.last)}${p.live ? ` · ${p.live} running` : ""}
-
${fmt.format(p.runs)}rollouts
${fmt.format(p.solved)}full marks
-
${p.runs ? `${Math.round((100 * p.done) / p.runs)}%` : "–"}finished
${fmt.format(p.public)}public
+
${fmt.format(p.runs)}rollouts
${fmt.format(p.scored)}graded
+
${p.runs ? `${Math.round((100 * p.done) / p.runs)}%` : "–"}finished
${fmt.format(p.public)}on Community
Models
${p.models.map(([m, n]) => `${esc(m.split("/")[1] || m)} ${n}`).join(", ")}
Datasets
${p.datasets.map(([x, n]) => `${esc(x)} ${n}`).join(", ")}
Since
${new Date(p.first * 1000).toLocaleDateString()}
@@ -184,36 +199,38 @@ async function people(body, { go }) { // ── environments: what trends, what's hidden, what's featured ──────────────── async function environments(body) { const d = await api("/api/admin/environments"); - if (!alive) return; - const st = { q: "", kind: "", flag: "", shown: 60 }; + if (!alive || !body.isConnected) return; + const st = { q: "", kind: "", flag: "", health: "", shown: 60 }; const cols = d.collections; const now = Date.now(); const isNew = (x) => x.created && now - Date.parse(x.created) < 14 * 864e5; const n = (f) => d.environments.filter(f).length; - body.innerHTML = `${head("Environments", "Pin to lead the trending row on Explore; hide to take off every listing (the page still opens by link); put in a collection to feature it.")} -
${icon("search", 16)}
+ body.innerHTML = `${head("Environments", "Pin to feature separately above the trending row on Explore; hide to take off every listing (the page still opens by link); put in a collection to feature it.")} +
${icon("search", 16)}
${[["", "All", d.environments.length], ["new", "New", n(isNew)], ["pinned", "Pinned", n((x) => x.pinned)], ["hidden", "Hidden", n((x) => x.hidden)], ["loose", "In no collection", n((x) => !x.collection)], ["unindexed", "Not indexed", n((x) => x.kind === "dataset" && !x.indexed)]] .map(([k, l, c]) => ``).join("")}
-
+

Discovery candidates are kept here for review. Only Spaces with fresh supported API evidence appear in the default explorer. Versions are declared dependencies, not runtime attestations.

`; const draw = () => { const q = st.q.toLowerCase(); const rows = d.environments.filter((x) => (!st.kind || x.kind === st.kind) && (!q || `${x.id} ${x.heading || ""} ${x.tags.join(" ")}`.toLowerCase().includes(q)) + && (!st.health || (x.kind === "space" && (st.health === "ready" ? x.ready : st.health === "candidate" ? !x.ready : x.api_status === st.health))) && (!st.flag || (st.flag === "new" ? isNew(x) : st.flag === "pinned" ? x.pinned : st.flag === "hidden" ? x.hidden : st.flag === "loose" ? !x.collection : x.kind === "dataset" && !x.indexed))) - .sort((a, b) => (b.pinned - a.pinned) || (b.trending * 1e9 + b.likes * 1e4 + Math.min(b.downloads, 9999)) - (a.trending * 1e9 + a.likes * 1e4 + Math.min(a.downloads, 9999))); + .sort((a, b) => (b.pinned - a.pinned) || (b.trending - a.trending) || (b.likes - a.likes) || (b.downloads - a.downloads)); $("#ad-envs", body).innerHTML = rows.length ? `
EnvironmentCollectionTrendingDownloads TasksRollouts
` + rows.slice(0, st.shown).map((x) => `
${x.kind === "space" ? `` : `${icon("database", 14)}`} - ${esc(x.id)} + ${esc(x.id)} + ${x.kind === "space" ? `${esc(x.ready ? (x.openenv ? "OpenEnv API checked" : x.framework + " API checked") : x.api_status || "Not checked")}${esc(x.declared_version || "Unknown version")}` : `${esc(x.framework)}`} ${x.pinned ? `${icon("flag", 11)}Pinned` : ""}${x.hidden ? `Hidden` : ""}${isNew(x) ? `New` : ""} ${fmt.format(x.trending || 0)}${x.kind === "dataset" ? compact.format(x.downloads) : `${compact.format(x.likes)} ♥`} - ${x.indexed?.tasks != null ? fmt.format(x.indexed.tasks) : x.kind === "dataset" ? `` : "–"} + ${x.indexed?.tasks != null ? fmt.format(x.indexed.tasks) : x.kind === "dataset" && x.framework === "harbor" ? `` : "–"} ${fmt.format(x.rollouts || 0)} - +
`).join("") : emptyState("search", "Nothing here", "Try another filter."); $("#ad-more", body).hidden = rows.length <= st.shown; @@ -228,14 +245,15 @@ async function environments(body) { if (action === "hide" || action === "unhide") x.hidden = action === "hide"; if (action === "feature" || action === "unfeature") x.collection = extra.collection || null; } - toast({ pin: "Pinned: it leads the trending row", unpin: "Unpinned", hide: "Hidden from listings", unhide: "Back in listings", feature: "Added to the collection", + toast({ pin: "Pinned: shown as an editorial pick", unpin: "Unpinned", hide: "Hidden from listings", unhide: "Back in listings", feature: "Added to the collection", unfeature: "Taken out of its collection", index: "Indexing started" }[action]); draw(); } catch (e) { toast(e.message); } }; let t; $("#ad-q", body).addEventListener("input", (e) => { clearTimeout(t); t = setTimeout(() => { st.q = e.target.value.trim(); st.shown = 60; draw(); }, 150); }); - $("#ad-kind", body).addEventListener("change", (e) => { st.kind = e.target.value; draw(); }); + $("#ad-health",body).addEventListener("change",e=>{st.health=e.target.value;st.shown=60;draw();}); + $("#ad-kind", body).addEventListener("change", (e) => { st.kind = e.target.value; st.shown=60; draw(); }); $("#ad-flag", body).addEventListener("click", (e) => { const b = e.target.closest("[data-flag]"); if (!b) return; st.flag = b.dataset.flag; body.querySelectorAll("#ad-flag [data-flag]").forEach((x) => x.setAttribute("aria-pressed", String(x === b))); draw(); }); $("#ad-more", body).addEventListener("click", () => { st.shown += 100; draw(); }); @@ -255,7 +273,7 @@ async function environments(body) { // ── collections ────────────────────────────────────────────────────────────── async function collections(body) { const d = await api("/api/admin/collections"); - if (!alive) return; + if (!alive || !body.isConnected) return; let cols = d.collections.map((c) => ({ ...c, ids: [...c.ids] })); let dirty = false; const draw = () => { @@ -298,13 +316,28 @@ async function collections(body) { draw(); } +function operations(o) { + const job=o.indexer, snap=o.snapshot, c=o.checks, tasks=o.task_catalog; + const state=job.stale ? "Update overdue" : job.state === "indexing" ? "Indexing" : job.state === "retrying" ? "Retry scheduled" : job.state ? "Waiting for next cycle" : "Waiting for first report"; + const when=v=>v ? ago(v) : "Not recorded"; + return `

${icon("refresh",15)}Catalog operations

Shared explorer state
+
Automatic indexing

${esc(state)}

Last success
${when(job.last_success)}
Next attempt
${job.next_at ? esc(new Date(job.next_at*1000).toLocaleString()) : "Not recorded"}
Snapshot
${snap.built_at ? esc(new Date(snap.built_at).toLocaleString()) : "No snapshot yet"}
${job.error || snap.error ? `

${esc(job.error || snap.error)}

` : ''}
+
Space API checks

${fmt.format(c.ready)} checked · ${fmt.format(c.openenv)} OpenEnv · ${fmt.format(c.candidates)} candidates

Latest check
${when(c.latest_at)}
Freshness
${c.fresh_seconds/60} minutes
Reported tasks
${fmt.format(tasks.tasks)} in ${fmt.format(tasks.spaces)} Spaces${tasks.partial_spaces ? ` · ${tasks.partial_spaces} partial catalogs` : ''}
+
${Object.entries(c.statuses).map(([v,n])=>`${esc(v)} · ${fmt.format(n)}`).join('')}
+

Task entries reported by Space APIs are separate from downloaded, searchable task records. Checks do not wake sleeping Spaces or certify grading quality. The explorer runs maintenance automatically while it is online.

+ ${c.versions.length ? `
Declared OpenEnv versions (${c.versions.length})
${c.versions.map(([v,n])=>`${esc(v)} · ${fmt.format(n)}`).join('')}
` : ''} +
`; +} + // ── indexes ────────────────────────────────────────────────────────────────── async function indexes(body) { const d = await api("/api/admin/indexes"); - if (!alive) return; + if (!alive || !body.isConnected) return; + body.indexEvents?.abort(); body.indexEvents = new AbortController(); const jobs = Object.entries(d.jobs); - body.innerHTML = `${head("Indexes", "Each dataset's index and pack: what makes its pages instant. Rebuild one after its dataset changes, or index a new one.")} -
${icon("database", 16)}
+ body.innerHTML = `${head("Indexing & API checks", "Automatic maintenance, fresh protocol evidence and dataset indexes across frameworks.", '')} + ${operations(d.operations)} +
${icon("database", 16)}
${jobs.length ? `

${spinner()}Indexing now

${jobs.map(([k, j]) => `
${esc(k)}
${esc(j.state)}${j.total ? ` · ${fmt.format(j.done || 0)} of ${fmt.format(j.total)}` : ""}${j.error ? ` · ${esc(j.error)}` : ""}
`).join("")}
` : ""} @@ -313,27 +346,28 @@ async function indexes(body) { ${x.built ? ago(x.built) : "–"}${x.current ? "" : ` old format`}${bytes(x.bytes)}${bytes(x.pack_bytes)} `).join("")}
`; const go = async (spec) => { - try { await api("/api/admin/environments", { method: "POST", body: { key: spec, action: "index" } }); toast(`Indexing ${spec}`); timer = setTimeout(() => alive && indexes(body), 1500); } + try { const result = await api("/api/admin/environments", { method: "POST", body: { key: spec, action: "index" } }); toast(result.job?.state === "done" ? "Index is current" : result.job?.state === "error" ? result.job.error || "Indexing failed" : `Indexing ${spec}`); timer = setTimeout(() => alive && body.isConnected && indexes(body), 1500); } catch (e) { toast(e.message); } }; + $("#ad-index-refresh",body).addEventListener("click",()=>indexes(body)); $("#ad-go", body).addEventListener("click", () => { const v = $("#ad-spec", body).value.trim(); if (v) go(v); }); $("#ad-spec", body).addEventListener("keydown", (e) => { if (e.key === "Enter") $("#ad-go", body).click(); }); - body.addEventListener("click", (e) => { const b = e.target.closest("[data-reindex]"); if (b) go(b.dataset.reindex); }); - if (jobs.length) timer = setTimeout(() => alive && indexes(body), 3000); + body.addEventListener("click", (e) => { const b = e.target.closest("[data-reindex]"); if (b) go(b.dataset.reindex); }, {signal:body.indexEvents.signal}); + if (jobs.length) timer = setTimeout(() => alive && body.isConnected && indexes(body), 3000); } // ── settings ───────────────────────────────────────────────────────────────── async function settings(body) { const s = await api("/api/admin/settings"); - if (!alive) return; + if (!alive || !body.isConnected) return; body.innerHTML = `${head("Settings", "They apply to the explorer within seconds.")}
RolloutsOff pauses new rollouts for everyone; running ones finish.
-
+
Rollouts at onceThe whole explorer, then each person.
/
-
Agents offeredWhat the run panel lists.
+
Harbor agent harnessesControls Harbor runs. MiMo and NeMo Gym use their own supported execution paths.
${s.all_agents.map((a) => ``).join("")}
AnnouncementOne line above every page of the explorer; empty for none.
@@ -362,7 +396,7 @@ async function settings(body) { // ── audit ──────────────────────────────────────────────────────────────────── async function audit(body) { const [d, ppl] = await Promise.all([api("/api/admin/audit"), api("/api/admin/people").catch(() => ({ people: [] }))]); - if (!alive) return; + if (!alive || !body.isConnected) return; const pics = Object.fromEntries(ppl.people.map((p) => [p.name, p.avatar])); const me = getSession().user; if (me) pics[me.name] = pics[me.name] || me.avatar; diff --git a/web/app.css b/web/app.css index abc1f823e64f002300c7b7a2952db84b06b83bee..cb10f3c93854c16bc150cfdd55a7719d5e31ebd6 100644 --- a/web/app.css +++ b/web/app.css @@ -501,7 +501,7 @@ mark { background: color-mix(in srgb, #f5c542 42%, transparent); color: inherit; .gated { margin-top: 14px; } /* the run panel */ -.tp-side { position: sticky; top: calc(var(--header-h) + 20px); display: grid; gap: 12px; } +.tp-side { position: sticky; top: calc(var(--header-h) + 20px); display: grid; gap: 12px; max-height: calc(100dvh - var(--header-h) - 32px); overflow: auto; scrollbar-width: thin; } /* a long unbreakable value (an image digest, a URL) wraps instead of widening the side column past the page */ .tp-side, .rp-side { min-width: 0; grid-template-columns: minmax(0, 1fr); } /* one column, the track's width: a long name inside wraps */ .runbox, .runbox .panel-b { min-width: 0; } @@ -738,7 +738,7 @@ a.rs-row:hover { background: var(--surface-2); } @media (max-width: 1024px) { :root { --gutter: 24px; } .tp-grid, .rp-grid { grid-template-columns: minmax(0, 1fr); } - .tp-side, .rp-side { position: static; } + .tp-side, .rp-side { position: static; max-height: none; overflow: visible; } .tp-side { order: -1; } .rp-side { grid-template-columns: repeat(auto-fit, minmax(min(100%, 320px), 1fr)); align-items: start; } .ex-stats { display: none; } diff --git a/web/js/community.js b/web/js/community.js index 5f5e63c148abf71ca8b309337ffb01fa38ec5567..2765724f1a9c931044b8e7181bae3980aa980729 100644 --- a/web/js/community.js +++ b/web/js/community.js @@ -1,233 +1,143 @@ -// Community: everyone's public rollouts, and the tasks nobody has tried yet, across every environment. Anonymous by -// construction: the API never returns who ran what. Filters and the view live in the URL, so a filtered page can be -// shared. (The MiMo RL Environment Explorer's community page, for every environment: MiMo's domains are groups.) -import { $, $$, api, esc, money, ago, rewardBadge, rewardText, statusPill, DOMAIN_NAME, sk, emptyState, fmt } from "./util.js"; -import { icon, DOMAIN_ICON } from "./icons.js"; -import { taskHref } from "./task.js"; +// Public, completed rollouts across all saved runners. Filters live in the URL. +import { $, $$, api, esc, money, ago, rewardBadge, DOMAIN_NAME, sk, emptyState, fmt } from "./util.js"; +import { icon } from "./icons.js"; -let alive = false; -export function unmount() { alive = false; } - -const REWARD = [["", "Any reward"], ["full", "Full marks"], ["partial", "Partial"], ["zero", "Zero"]]; -const THINK = { default: "model default", none: "off", low: "low", medium: "medium", high: "high" }; -const SORT = [["new", "Newest"], ["reward_desc", "Highest reward"], ["reward_asc", "Lowest reward"], ["cost_asc", "Cheapest"], ["cost_desc", "Most expensive"]]; -const RUNNER = { harbor: "Harbor", mimo: "MiMo harness" }; -const DOM_BY_NAME = Object.fromEntries(Object.entries(DOMAIN_NAME).map(([k, v]) => [v, k])); -const short = (m) => (m || "").split("/")[1] || m || ""; -const num = (x) => (x == null ? "–" : x.toFixed(2)); -const GROUPS = 6; // columns of the leaderboard, buttons of the filter -const enc = (s) => s.split("/").map(encodeURIComponent).join("/"); - -export async function mount(el, { qs }) { - alive = true; - const p = qs || new URLSearchParams(); - const f = { view: p.get("view") === "tasks" ? "tasks" : "rollouts", group: p.get("group") || DOMAIN_NAME[p.get("domain")] || "", dataset: p.get("dataset") || "", - model: p.get("model") || "", served: p.get("served") || "", judge: p.get("judge") || "", reward: p.get("reward") || "", - thinking: p.get("thinking") || "", runner: p.get("runner") || "", sort: p.get("sort") || "new", q: p.get("q") || "", has: p.get("has") || "none" }; - el.innerHTML = `
-

Community

Public rollouts from everyone, shown without who ran them. Compare models on the same tasks, read their traces, or pick a task nobody has tried yet. Yours are under My rollouts.

-
-
-
-

Public rollouts may later be released as an open dataset (for example SFT traces) for research.

`; - const sync = () => { - const q = new URLSearchParams(Object.entries(f).filter(([k, v]) => v && !(k === "view" && v === "rollouts") && !(k === "sort" && v === "new") && !(k === "has" && v === "none"))); - history.replaceState(null, "", "/community" + (q.toString() ? "?" + q : "")); - $$("#cm-view [data-view]", el).forEach((b) => b.setAttribute("aria-pressed", String(b.dataset.view === f.view))); - }; - $("#cm-view", el).addEventListener("click", (e) => { const b = e.target.closest("[data-view]"); if (b) { f.view = b.dataset.view; render(); } }); - const render = () => { - sync(); - const body = $("#cm-body", el), fresh = body.cloneNode(false); // a fresh container: the old view's listeners go with it - body.replaceWith(fresh); - (f.view === "tasks" ? tasksView : rolloutsView)(el, f, sync); - }; - render(); +let current = null; +export function unmount() { + if (current) { current.controller?.abort(); current.events.abort(); clearTimeout(current.timer); } + current = null; } - -const groupIcon = (g) => (DOM_BY_NAME[g] ? icon(DOMAIN_ICON[DOM_BY_NAME[g]], 11) : icon("database", 11)); -const groupColor = (g) => (DOM_BY_NAME[g] ? `var(--c-${DOM_BY_NAME[g]})` : "var(--c-hub)"); +const REWARD = [["", "Any reward"], ["full", "Reward ≥ 0.999"], ["partial", "0 < reward < 0.999"], ["zero", "Reward ≤ 0"]]; +const SORT = [["new", "Newest"], ["reward_desc", "Highest reward"], ["reward_asc", "Lowest reward"], ["cost_asc", "Cheapest"], ["cost_desc", "Most expensive"]]; +const THINK = { default: "model default", none: "off", low: "low", medium: "medium", high: "high" }; +const RUNNER = { harbor: "Harbor", mimo: "MiMo", "nemo-gym": "NeMo Gym", openenv: "OpenEnv", verifiers: "Verifiers", verl: "verl", skyrl: "SkyRL", mcp: "MCP", ors: "ORS" }; +const runnerName = (value) => RUNNER[value] || value || "Harbor"; +const short = (value) => (value || "").split("/").at(-1) || "Unknown model"; +const num = (value) => value == null ? "–" : new Intl.NumberFormat("en-US", { maximumFractionDigits: 4 }).format(value); +const KEYS = ["dataset", "runner", "group", "model", "served", "judge", "reward", "thinking", "q"]; +const LABEL = { model: short, served: (v) => "served by " + v, judge: (v) => "judge " + short(v), thinking: (v) => "thinking " + (THINK[v] || v), + reward: (v) => (REWARD.find(([k]) => k === v) || [v, v])[1], q: (v) => `“${v}”`, group: (v) => v, dataset: (v) => v, runner: runnerName }; function row(r) { - const thumb = r.shot ? `` : ""; - return ` - ${rewardBadge(r.reward, r.status)}${esc(r.title)}
${groupIcon(r.group)} ${esc(r.group)} - · ${esc(RUNNER[r.runner || "harbor"] || r.runner)}${r.runner === "mimo" ? "" : ` · ${esc(r.harness || "")}`} · ${esc(r.endpoint ? "own endpoint" : r.provider || "auto")}${r.judge ? ` · judge ${esc(short(r.judge))}` : ""} - ${thumb} - ${esc(r.endpoint ? r.model || "custom endpoint" : short(r.model))}${money((r.cost || {}).total)} + const title = r.title || r.path || r.task_id || "Rollout"; + const details = [r.dataset, runnerName(r.runner), r.domain && DOMAIN_NAME[r.domain], + r.harness && !["mimo", "nemo-gym"].includes(r.runner) ? r.harness : null, + r.endpoint ? "own endpoint" : r.provider, r.judge ? `judge ${short(r.judge)}` : null].filter(Boolean).join(" · "); + return ` + ${rewardBadge(r.reward, r.status)}${esc(title)}
${esc(details)} + ${r.shot ? `` : ""} + ${esc(short(r.model))}${money(r.cost?.total)} ${ago(r.created_at)}${icon("chevronRight", 14)}`; } -function groupFilter(groups, current) { - const top = groups.slice(0, GROUPS), rest = groups.slice(GROUPS); - return `
${[["", "All"], ...top.map(([g]) => [g, g])].map(([k, l]) => - ``).join("")}
- ${rest.length ? `` : ""}`; +function modelResults(results) { + return `
+ ${results.map((r) => ` + + + `).join("")}
ModelEnvironmentRunnerMetricRunsMean reward
${esc(r.served)} + ${esc([DOMAIN_NAME[r.domain] || r.domain, r.revision ? `revision ${r.revision.slice(0, 8)}` : ""].filter(Boolean).join(" · "))}${esc(runnerName(r.runner))}${esc(r.metric)}${r.judge ? `judge ${esc(short(r.judge))}` : ""}${fmt.format(r.runs)}${num(r.mean)}
`; } -function rolloutsView(el, f, sync) { - const body = $("#cm-body", el); - body.innerHTML = ` +export async function mount(el, { qs }) { + unmount(); + const page = current = { controller: null, events: new AbortController(), timer: null, sequence: 0, pending: false }; + const p = qs || new URLSearchParams(); + const f = Object.fromEntries(KEYS.map((k) => [k, p.get(k) || ""])); + f.dataset ||= p.get("d") || ""; + f.group ||= DOMAIN_NAME[p.get("domain")] || ""; + f.sort = SORT.some(([k]) => k === p.get("sort")) ? p.get("sort") : "new"; + // Legacy Tasks URLs become rollout filters; no implicit MiMo dataset selection. + const sync = () => { + const q = new URLSearchParams(Object.entries(f).filter(([k, v]) => v && !(k === "sort" && v === "new"))); + history.replaceState(null, "", "/community" + (q.size ? `?${q}` : "")); + }; + el.innerHTML = `
+

Community

Public, graded rollouts across Harbor, MiMo, NeMo Gym and other supported runners. Explore their traces and compare results within an environment. Your public and private runs are in My rollouts.

${sk.line(60, 22)}
-

Leaderboard

mean reward over the rollouts shown · best per column in bold
-
${sk.lines(90, 80, 85)}
-
-
-

Rollouts

-
${sk.lines(95, 90, 92, 88)}
-
`; +
+

Reward scales differ by environment. Values below use each rollout’s recorded scoring metric.

+

Rollouts

+
${sk.lines(95, 90, 92, 88)}
+
+ +

Public rollouts hide account identities and may later be released as an open dataset for research.

`; let offset = 0; - const LABEL = { model: (v) => short(v), served: (v) => "served by " + v, judge: (v) => "judge " + short(v), thinking: (v) => "thinking " + (THINK[v] || v), - reward: (v) => (REWARD.find(([k]) => k === v) || [v, v])[1].toLowerCase(), q: (v) => `“${v}”`, group: (v) => v, dataset: (v) => v, - runner: (v) => RUNNER[v] || v }; - const KEYS = ["group", "dataset", "model", "served", "judge", "reward", "thinking", "runner", "q"]; - const load = async (append = false) => { - const q = new URLSearchParams({ limit: "40", offset: String(append ? offset : 0) }); - for (const k of [...KEYS, "sort"]) if (f[k]) q.set(k, f[k]); - let d; - try { d = await api(`/api/community?${q}`); } catch (e) { $("#cm-rows", el).innerHTML = `

${esc(e.message)}

`; return; } - if (!alive) return; - const s = d.stats, fc = d.facets; - if (!append) { - const all = fc.group.reduce((n, [, c]) => n + c, 0); - $("#cm-top", el).innerHTML = ""; - const fig = (v, l) => `
${v}${l}
`; - $("#cm-figs", el).innerHTML = fig(fmt.format(all), "public rollouts") + fig(fmt.format(d.tasks), "tasks tried") + fig(fmt.format(fc.dataset.length), "environments") - + fig(fc.model.length, "models") + fig(num(s.mean), "mean reward") + fig(s.scored ? `${Math.round(s.full / s.scored * 100)}%` : "–", "full marks"); - const cols = fc.group.slice(0, GROUPS).map(([g]) => g); - const mx = s.matrix || {}; - const best = Object.fromEntries(cols.map((k) => [k, Math.max(-1, ...s.by_model.map((m) => ((mx[m.model] || {})[k] || {}).mean ?? -1))])); - const bestAll = Math.max(-1, ...s.by_model.map((m) => m.mean)); - $("#cm-board", el).innerHTML = s.by_model.length ? `
- ${cols.map((k) => ``).join("")} - ${s.by_model.map((m, i) => ` - - ${cols.map((k) => { const c = (mx[m.model] || {})[k]; - return c ? `` - : ``; }).join("")}`).join("")}
#ModelRunsMeanFull${esc(k.length > 16 ? k.slice(0, 15) + "…" : k)}
${i + 1}${m.custom ? ' own endpoint' : ""}${m.runs}${num(m.mean)}${m.full}–
` - : `

No scored rollouts for these filters.

`; - const opt = (key, label, list, fmtv = (x) => x) => ``; - $("#cm-filters", el).innerHTML = `${groupFilter(fc.group, f.group)} -
${icon("search", 15)}
- ${opt("model", "Any model", fc.model, short)} - -
${icon("filter", 14)}More
- ${opt("dataset", "Any dataset", fc.dataset)}${opt("runner", "Any runner", fc.runner, (v) => RUNNER[v] || v)} - ${opt("served", "Served by: any", fc.served)}${opt("judge", "Any judge", fc.judge, short)}${opt("thinking", "Any thinking", fc.thinking, (v) => "thinking " + (THINK[v] || v))}
- `; - const active = KEYS.filter((k) => f[k]); - $("#cm-active", el).innerHTML = active.length ? `Filtered by` + active.map((k) => ``).join("") - + `` : ""; - $("#cm-rows", el).innerHTML = ""; - } - $("#cm-rows", el).insertAdjacentHTML("beforeend", d.runs.map(row).join("")); - const n = $$("#cm-rows .runrow", el).length; - offset = n; - $("#cm-count", el).textContent = d.total ? `${n} of ${d.total}` : ""; - $("#cm-more", el).hidden = n >= d.total; - $("#cm-more span", el).textContent = `Showing ${n} of ${d.total}`; - if (!d.total && !append) { - $("#cm-rows", el).innerHTML = emptyState(KEYS.some((k) => f[k]) ? "filter" : "globe", KEYS.some((k) => f[k]) ? "Nothing matches" : "Nothing shared yet", - KEYS.some((k) => f[k]) ? "Try fewer filters, or switch to Tasks to find one nobody has run." : "Run a task and keep it public: it shows here, without your name, once it's graded."); - } + const seen = new Set(); + const option = (key, label, entries = [], labelOf = (v) => v) => { + const list = entries.slice(); + if (f[key] && !list.some(([v]) => v === f[key])) list.push([f[key], 0]); + return ``; }; - let t; - const reload = () => { sync(); load(); }; - // a screenshot that won't load just disappears (no inline onerror: the CSP forbids it) - body.addEventListener("error", (e) => { if (e.target.classList?.contains("cm-shot")) e.target.remove(); }, true); - body.addEventListener("click", (e) => { - const b = e.target.closest("[data-group]"); - if (b) { f.group = b.dataset.group; reload(); return; } - const pm = e.target.closest("[data-pick-model]"); - if (pm) { f.model = pm.dataset.pickModel; reload(); return; } - const cell = e.target.closest("[data-cell]"); - if (cell) { const i = cell.dataset.cell.lastIndexOf("|"); f.model = cell.dataset.cell.slice(0, i); f.group = cell.dataset.cell.slice(i + 1); reload(); return; } - const u = e.target.closest("[data-unset]"); - if (u) { f[u.dataset.unset] = ""; reload(); return; } - if (e.target.closest("#cm-clear")) { for (const k of KEYS) f[k] = ""; reload(); return; } - if (e.target.closest("#cm-more button")) load(true); - }); - body.addEventListener("change", (e) => { const s = e.target.closest("[data-f]"); if (s) { f[s.dataset.f] = s.value; reload(); } }); - body.addEventListener("input", (e) => { if (e.target.id === "cm-q") { clearTimeout(t); t = setTimeout(() => { f.q = e.target.value.trim(); reload(); }, 250); } }); - load(); -} - -// ── tasks: one environment's, with how many public rollouts each has ──────── -async function tasksView(el, f, sync) { - const body = $("#cm-body", el); - body.innerHTML = `
${sk.line(40, 30)}

${icon("grid", 15)}Tasks

-
${sk.lines(95, 90, 92, 88, 94)}
-
`; - let envs; - try { envs = (await api("/api/community?limit=1")).facets.dataset.map(([d]) => d); } catch { envs = []; } - const MIMO = "XiaomiMiMo/MiMo-V2.6-RL-oss"; - if (!envs.includes(MIMO)) envs.push(MIMO); - const ds = f.dataset && envs.includes(f.dataset) ? f.dataset : envs.includes(MIMO) ? MIMO : envs[0]; - f.dataset = ds; - let cards = [], tried = {}; - try { - const [sum, t] = await Promise.all([api(`/api/env/${enc(ds)}`), api(`/api/community/tasks?dataset=${encodeURIComponent(ds)}`)]); - tried = t.tasks; - cards = sum.inline ? (await api(`/api/env/${enc(ds)}/tasks?all=1`)).cards - : Object.entries(tried).map(([ref, x]) => ({ ref, title: x.title || ref, lead: null, sub: [], chips: [] })); // a paged dataset: the tasks that were run - } catch (e) { if (alive) $("#ct-rows", el).innerHTML = `

${esc(e.message)}

`; return; } - if (!alive) return; - const leads = [...new Set(cards.map((c) => c.lead).filter(Boolean))]; - if (f.group && !leads.includes(f.group)) f.group = ""; - let list = [], shown = 0; - const seed = Math.random(); - const rank = (id) => { let h = 2166136261; for (let i = 0; i < id.length; i++) h = Math.imul(h ^ id.charCodeAt(i), 16777619); return ((h >>> 0) ^ (seed * 4294967295)) >>> 0; }; - const filter = () => { - const q = f.q.toLowerCase(); - list = cards.filter((c) => (!f.group || c.lead === f.group) && (f.has === "all" || (f.has === "none" ? !tried[c.ref] : !!tried[c.ref])) - && (!q || `${c.title} ${c.ref}`.toLowerCase().includes(q))); - if (f.has === "some") list.sort((a, b) => (tried[b.ref].last || 0) - (tried[a.ref].last || 0)); - else list.sort((a, b) => rank(a.ref) - rank(b.ref)); // shuffled, so people don't all pick the same untried task - shown = 0; - $("#ct-rows", el).innerHTML = ""; - more(); - const none = cards.filter((c) => (!f.group || c.lead === f.group) && !tried[c.ref]).length; - $("#cm-top", el).innerHTML = `
${fmt.format(Object.keys(tried).length)}tasks with public rollouts
-
${fmt.format(none)}${f.group ? esc(f.group) + " tasks" : "tasks"} not tried yet
`; + const renderHead = (d) => { + const s = d.stats, fc = d.facets; + const fig = (v, label) => `
${fmt.format(v)}${label}
`; + $("#cm-figs", el).innerHTML = fig(d.total, "public rollouts") + fig(s.datasets, "environments") + fig(s.runners, "runners") + fig(s.models, "models"); + $("#cm-result-panel", el).hidden = !s.results.length; + $("#cm-board", el).innerHTML = s.results.length ? modelResults(s.results) : ""; + const focus = document.activeElement, focusedQuery = focus?.id === "cm-q", focusedFilter = focus?.dataset?.f; + const selection = focusedQuery ? [focus.selectionStart, focus.selectionEnd] : null; + const moreOpen = $(".cm-more", el)?.open; + $("#cm-filters", el).innerHTML = `
${icon("search", 15)}
+ ${option("runner", "Any runner", fc.runner, runnerName)}${option("dataset", "Any environment", fc.dataset)}${option("model", "Any model", fc.model, short)} + +
${icon("filter", 14)}More
+ ${option("group", "Any group", fc.group)}${option("served", "Any provider", fc.served)}${option("judge", "Any judge", fc.judge, short)}${option("thinking", "Any thinking", fc.thinking, v => THINK[v] || v)}
+ `; + if (focusedQuery) { const input = $("#cm-q", el); input.focus({preventScroll:true}); input.setSelectionRange(...selection); } + else if (focusedFilter) $$('[data-f]', el).find(x => x.dataset.f === focusedFilter)?.focus({preventScroll:true}); + const active = KEYS.filter(k => f[k]); + $("#cm-active", el).innerHTML = active.length ? 'Filtered by' + active.map(k => ``).join("") + '' : ""; }; - const more = () => { - const next = list.slice(shown, shown + 40); - $("#ct-rows", el).insertAdjacentHTML("beforeend", next.map((c) => { - const x = tried[c.ref]; - return ` - ${x ? `= 0.999 ? "full" : x.mean > 0 ? "part" : "zero"}" title="mean reward">${x.mean == null ? "–" : rewardText(x.mean)}` : `${icon("play", 11)}New`} - ${esc(c.title)}
${c.icon ? icon(c.icon, 11) : ""} ${esc([c.lead, ...(c.sub || [])].filter(Boolean).join(" · ") || c.ref)} - ${x ? `${x.runs} rollout${x.runs === 1 ? "" : "s"}` : "no rollouts yet"}${x ? "best " + rewardText(x.best) : ""} - ${icon("chevronRight", 14)}`; - }).join("")); - shown += next.length; - $("#ct-count", el).textContent = `${fmt.format(list.length)} tasks`; - $("#ct-more", el).hidden = shown >= list.length; - $("#ct-more span", el).textContent = `Showing ${fmt.format(shown)} of ${fmt.format(list.length)}`; - if (!list.length) $("#ct-rows", el).innerHTML = emptyState("check", f.has === "none" ? "Every task here has a public rollout" : "No tasks match", "Try another filter."); + const load = async (append = false) => { + if (append && page.pending) return; + page.controller?.abort(); + const controller = page.controller = new AbortController(), sequence = ++page.sequence; + page.pending = true; + $("#cm-more button", el).disabled = true; + $("#cm-rows", el).setAttribute("aria-busy", "true"); + $("#cm-error", el).innerHTML = ""; + const q = new URLSearchParams({limit:"40",offset:String(append ? offset : 0)}); + for (const [key, value] of Object.entries(f)) if (value) q.set(key, value); + try { + const d = await api(`/api/community?${q}`, {signal:controller.signal}); + if (current !== page || sequence !== page.sequence) return; + if (!append) { renderHead(d); seen.clear(); offset = 0; $("#cm-rows", el).innerHTML = ""; } + const next = d.runs.filter(r => { if (seen.has(r.id)) return false; seen.add(r.id); return true; }); + $("#cm-rows", el).insertAdjacentHTML("beforeend", next.map(row).join("")); + offset += d.runs.length; + $("#cm-count", el).textContent = d.total ? `${fmt.format(seen.size)} of ${fmt.format(d.total)}` : ""; + $("#cm-more", el).hidden = offset >= d.total || !d.runs.length; + $("#cm-more span", el).textContent = `Showing ${fmt.format(seen.size)} of ${fmt.format(d.total)}`; + if (!d.total) $("#cm-rows", el).innerHTML = emptyState("globe", KEYS.some(k => f[k]) ? "No rollouts match" : "No public rollouts yet", + KEYS.some(k => f[k]) ? "Clear a filter to see more public rollouts." : "Completed, graded runs appear here when shared publicly. Private runs remain in My rollouts.", + 'My rolloutsExplore environments'); + } catch (e) { + if (current === page && sequence === page.sequence && e.name !== "AbortError") { + if (!append) $("#cm-rows", el).innerHTML = ""; + $("#cm-error", el).innerHTML = `

Couldn't load rollouts: ${esc(e.message)}

`; + } + } finally { + if (current === page && sequence === page.sequence) { page.pending = false; $("#cm-more button", el).disabled = false; $("#cm-rows", el).setAttribute("aria-busy", "false"); } + } }; - $("#ct-filters", el).innerHTML = ` - - ${leads.length > 1 && leads.length <= 8 ? `
${[["", "All"], ...leads.map((l) => [l, l])].map(([k, l]) => ``).join("")}
` : ""} -
${[["none", "No rollouts yet"], ["some", "Has rollouts"], ["all", "All tasks"]].map(([k, l]) => ``).join("")}
-
${icon("search", 15)}
`; - let t; - body.addEventListener("click", (e) => { - const b = e.target.closest("[data-lead]"); - if (b) { f.group = b.dataset.lead; $$("[data-lead]", body).forEach((x) => x.setAttribute("aria-pressed", String(x === b))); sync(); filter(); return; } - const h = e.target.closest("[data-has]"); - if (h) { f.has = h.dataset.has; $$("[data-has]", body).forEach((x) => x.setAttribute("aria-pressed", String(x === h))); sync(); filter(); return; } - if (e.target.closest("#ct-more button")) more(); - }); - body.addEventListener("change", (e) => { - if (e.target.id !== "ct-ds") return; - f.dataset = e.target.value; f.group = ""; sync(); - body.replaceWith(body.cloneNode(false)); // the next environment's view, without this one's listeners - tasksView(el, f, sync); - }); - body.addEventListener("input", (e) => { if (e.target.id === "ct-q") { clearTimeout(t); t = setTimeout(() => { f.q = e.target.value.trim(); sync(); filter(); }, 200); } }); - sync(); - filter(); + const reload = () => { clearTimeout(page.timer); sync(); load(); }; + el.addEventListener("error", e => { if (e.target.classList?.contains("cm-shot")) e.target.remove(); }, {capture:true, signal:page.events.signal}); + el.addEventListener("click", e => { + const unset = e.target.closest("[data-unset]"), model = e.target.closest("[data-pick-model]"), dataset = e.target.closest("[data-pick-dataset]"), retry = e.target.closest("[data-retry]"); + if (unset) { f[unset.dataset.unset] = ""; reload(); } + else if (model) { f.model = model.dataset.pickModel; reload(); } + else if (dataset) { f.dataset = dataset.dataset.pickDataset; reload(); } + else if (retry) load(retry.dataset.retry === 'more'); + else if (e.target.closest("#cm-clear")) { for (const key of KEYS) f[key] = ""; reload(); } + else if (e.target.closest("#cm-more button")) load(true); + }, {signal:page.events.signal}); + // Tie every listener to this mount, even when the router reuses the main node. + const change = e => { const select = e.target.closest("[data-f]"); if (select) { f[select.dataset.f] = select.value; reload(); } }; + const input = e => { if (e.target.id !== "cm-q") return; f.q = e.target.value.trim(); page.controller?.abort(); ++page.sequence; clearTimeout(page.timer); page.timer = setTimeout(reload, 250); }; + el.addEventListener("change", change, {signal:page.events.signal}); + el.addEventListener("input", input, {signal:page.events.signal}); + sync(); load(); } diff --git a/web/js/compare.js b/web/js/compare.js index dccb150c334466954aa7fadf3fbddd9e2278fcc5..23f5e1359a846f5adcdca2ea5ed1c3f6dd8a5b59 100644 --- a/web/js/compare.js +++ b/web/js/compare.js @@ -127,7 +127,9 @@ function fromTrajectory(run, traj) { } if (run.reward != null) { const named = Object.entries(run.rewards || {}); - push({ kind: "checks", reward: run.reward, summary: "", checks: named.map(([k, v]) => ({ id: k, score: v, passed: v >= 0.999 ? true : v <= 0 ? false : null })) }); + // Harbor allows arbitrary named metrics (latency, loss, counts, signed scores). + // Their values do not imply pass/fail without the verifier's own criteria. + push({ kind: "checks", reward: run.reward, summary: "", checks: named.map(([k, v]) => ({ id: k, score: v, passed: null })) }); } return out; } diff --git a/web/js/count-up.js b/web/js/count-up.js new file mode 100644 index 0000000000000000000000000000000000000000..73f33d5399685b4a9ce283e04b244c577864003c --- /dev/null +++ b/web/js/count-up.js @@ -0,0 +1,15 @@ +// The accessible label always holds the final count. Only the visual number +// animates, once per change, and detached views stop requesting frames. +export function countUp(node, target, format, from = 0) { + node.textContent = format(target); + if (from === target || matchMedia("(prefers-reduced-motion: reduce)").matches) return; + node.textContent = format(from); + const start = performance.now(); + const frame = (now) => { + if (!node.isConnected) return; + const t = Math.min(1, (now - start) / 750); + node.textContent = format(Math.round(from + (target - from) * (1 - (1 - t) ** 3))); + if (t < 1) requestAnimationFrame(frame); + }; + requestAnimationFrame(frame); +} diff --git a/web/js/data.js b/web/js/data.js index 776a9baa90b70c69eb15e0a6644b561736d4abf6..ee823e43a81a2bc114502df3ae3f91eb3f107a76 100644 --- a/web/js/data.js +++ b/web/js/data.js @@ -23,5 +23,4 @@ export function myDatasets() { } export function forgetMine() { mine = null; } -// trending first (the Hub's score, the same scale for datasets and Spaces), then likes, then downloads -export const trendingKey = (d) => d.trending * 1e9 + d.likes * 1e4 + Math.min(d.downloads || 0, 9999); +// Ranking is centralized in /api/search so list, filters and individual ranks agree. diff --git a/web/js/env.js b/web/js/env.js index 3bf7020131695b05a56445f180134b3f44c2b3ee..7451f61a7331880d361d7cbf0e0a794ffd20d1f9 100644 --- a/web/js/env.js +++ b/web/js/env.js @@ -3,8 +3,9 @@ // inline every task at once (a Harbor index): search and filters run here as you type, with tiles and a map // paged a page at a time (dataset rows): subsets, search and filters run on the server // A Harbor dataset is indexed the first time it's opened; the page shows the indexing's progress meanwhile. -import { $, api, esc, fmt, ago, sk, emptyState, toast, spinner, nav, setMeta } from "./util.js"; +import { $, api, esc, fmt, ago, sk, emptyState, toast, spinner, nav, setMeta, ownerLink } from "./util.js"; import { icon } from "./icons.js"; +import { supportPanel } from "./support.js"; import { blocks, loadRenderers } from "./blocks.js"; import { taskHref } from "./task.js"; import { typeset } from "./math.js"; @@ -113,9 +114,11 @@ async function load(el) { } async function loadCards() { - try { st.cards = (await api(`${envApi(st.spec)}/tasks?all=1`)).cards; } catch (e) { st.cards = []; toast(e.message); } - const n = st.cards.length; - st.cards.forEach((c) => { c._blob = `${c.title} ${c.ref} ${c.id || ""} ${c.brief || ""} ${c.text || ""}`.toLowerCase(); }); + const state = st; + try { state.cards = (await api(`${envApi(state.spec)}/tasks?all=1`)).cards; } catch (e) { state.cards = []; if (state === st) toast(e.message); } + if (state !== st) return; + const n = state.cards.length; + state.cards.forEach((c) => { c._blob = `${c.title} ${c.ref} ${c.id || ""} ${c.brief || ""} ${c.text || ""}`.toLowerCase(); }); // what every task shares says nothing on a card const count = new Map(); st.cards.forEach((c) => c.chips.forEach((x) => count.set(x, (count.get(x) || 0) + 1))); @@ -140,7 +143,7 @@ function renderHead(el) {
${c ? `${icon(c.icon, 13)}${esc(c.group)}` : ""} ${esc(s.env.framework)}${(i.badges || []).map((b) => `${esc(b)}`).join("")}
-

${esc(org)}/${esc(name)}

+

${ownerLink(org, "/")}${esc(name)}

${lede ? `

${esc(lede)}

` : ""}
${n != null ? `${icon("list", 14)}${fmt.format(n)} task${n === 1 ? "" : "s"}` : s.subsets?.length > 1 ? `${icon("columns", 14)}${fmt.format(s.subsets.length)} subsets` : ""} @@ -151,7 +154,7 @@ function renderHead(el) { ${icon("flag", 14)}${esc((i.sha || "").slice(0, 8))} ${icon("external", 14)}On the Hub -
`; +
${supportPanel(s.support)}`; $("#copy", el).addEventListener("click", () => navigator.clipboard?.writeText(location.href).then(() => toast("Link copied"))); } diff --git a/web/js/home.js b/web/js/home.js index 428008c6a1e3280c739acacaddab275867ab5f95..72774e5f1a62e795be9890303f6c54140ab552e9 100644 --- a/web/js/home.js +++ b/web/js/home.js @@ -2,21 +2,22 @@ // through, then everything: Harbor task datasets and OpenEnv Spaces, by collection, searchable and filterable. // The server filters, counts and sorts (/api/search, over the catalog snapshot: app/search_api.py); the page asks // for one page of cards at a time, and the first request also brings the header numbers and the trending row. -import { $, api, esc, fmt, ago, sk, progress, emptyState, toast, openDialog, closeModal, storage, nav } from "./util.js"; +import { $, api, esc, fmt, ago, sk, progress, emptyState, toast, openDialog, closeModal, storage, nav, setMeta, ownerLink } from "./util.js"; import { icon } from "./icons.js"; import { myDatasets } from "./data.js"; +import { countUp } from "./count-up.js"; const PAGE = 40, TRENDING = 24; const compact = new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }); const enc = (s) => s.split("/").map(encodeURIComponent).join("/"); const href = (d) => `/${d.kind === "space" ? "s" : "d"}/${enc(d.id)}`; -const SORTS = [["trending", "trending on the Hub first"], ["downloads", "most downloaded first"], ["likes", "most liked first"], +const SORTS = [["trending", "Hub interest, then likes and downloads"], ["downloads", "most downloaded first"], ["likes", "most liked first"], ["tasks", "most tasks first"], ["rollouts", "most rollouts first"], ["updated", "recently updated first"], ["new", "newest first"]]; const time = (iso) => (iso ? Date.parse(iso) : 0); // what kind of environment each is: Harbor task folders, an OpenEnv (or other) Space, or rows a processor reads -// Spaces: OpenEnv servers (a manifest or the tag), ORS servers, other environment servers; datasets: by the Hub's +// Spaces: API-checked OpenEnv servers, unverified discovery candidates, other environment servers; datasets: by the Hub's // framework tags, Harbor whenever its index found task folders (the server works it out: `fw` on each card) -const KINDS = [["all", "Everything", "All"], ["harbor", "Harbor datasets", "Harbor"], ["openenv", "OpenEnv Spaces", "OpenEnv"], +const KINDS = [["all", "Everything", "All"], ["harbor", "Harbor datasets", "Harbor"], ["openenv", "OpenEnv · API checked", "OpenEnv"], ["unverified-space", "Unverified Space candidates", "Unverified Spaces"], ["other-space", "Other environment Spaces", "Other Spaces"], ["verifiers", "Verifiers", "Verifiers"], ["nemo-gym", "NeMo Gym", "NeMo Gym"], ["openenv-data", "OpenEnv datasets", "OpenEnv data"], ["verl", "verl and SkyRL", "verl"], ["rows", "Other RL datasets", "Other"]]; @@ -28,12 +29,21 @@ const FW = { harbor: ["database", "Harbor", "dataset", "A dataset of tasks in Ha const OTHER = "Not in a collection"; const SIZE_ORDER = ["Under 100", "100 to 1,000", "1,000 or more", "Not indexed yet"]; const MINE = { id: "mine", group: "Your datasets", icon: "user", color: null, about: "yours and your organizations', private ones too" }; -const FACETS = [["type", "Labelled on the Hub as"], ["size", "Tasks"], ["stage", "Space status"], ["mcp", "Tools over MCP"], - ["oe", "OpenEnv version"], ["tags", "Tags"]]; +const FACETS = [["health", "OpenEnv API availability"], ["mode", "OpenEnv interface"], ["oe", "OpenEnv version (declared)"], + ["oe_source", "Version evidence"], ["tools", "MCP tool discovery"], ["type", "Labelled on the Hub as"], ["size", "Tasks"], + ["stage", "Space status at snapshot"], ["mcp", "MCP metadata"], ["evidence", "OpenEnv evidence"], ["tags", "Tags"]]; +const FACET_HELP = { + health: "API checked means running with healthy OpenEnv endpoints checked within the last hour. Episodes and rewards are not tested.", + mode: "Simulation exposes reset, step and state. MCP service exposes tools without episode control routes.", + oe: "Repository dependency or Hub tag. This does not confirm the installed runtime version. Ranges and Git references are kept as declared.", + tools: "Tools returned by the server within the last hour. Discovery does not test tool execution.", +}; +const versionGroup = (v) => /^\d/.test(v) ? 0 : /^[<>=!~]/.test(v) ? 1 : v.startsWith("git:") ? 2 : v === "Unknown" ? 4 : 3; -let root = null, COLLECTIONS = [], COLL = {}, STATS = null, TREND = null, MINE_N = 0; -const state = { q: "", coll: null, kind: "all", sel: {}, sort: "trending", mine: false }; +let root = null, COLLECTIONS = [], COLL = {}, STATS = null, TREND = null, FEATURED = [], PROMOTED = [], SNAPSHOT = null, RANKING = null, MINE_N = 0; +const state = { q: "", owner: null, coll: null, kind: "all", sel: {}, sort: "trending", mine: false, candidates: false }; let res = null, rows = [], page = 1, seq = 0, ctrl = null, typing = null; +let refreshTimer = null, refreshing = false, lastAnswer = 0; const known = new Map(); // every card seen, by key: what the quick look opens const expanded = new Set(), facetOpen = new Map(); @@ -52,7 +62,9 @@ function fString() { } function query({ page: pg = 1, facets = true, trending = 0 } = {}) { const p = new URLSearchParams(); + p.set("scope", state.candidates || state.kind === "unverified-space" ? "all" : "ready"); if (state.q) p.set("q", state.q); + if (state.owner) p.set("owner", state.owner); if (state.kind !== "all") p.set("kind", state.kind); if (state.coll) p.set("collection", state.coll); if (state.sort !== "trending") p.set("sort", state.sort); @@ -72,10 +84,16 @@ function apply(d) { if (MINE_N) COLL.mine = { ...MINE, ids: [] }; } if (d.stats) STATS = d.stats; + if (d.snapshot) SNAPSHOT = d.snapshot; + if (d.ranking) RANKING = d.ranking; + if (d.featured) FEATURED = d.featured; + if (d.promoted) { PROMOTED = d.promoted; remember(PROMOTED); } if (d.trending) { TREND = d.trending; remember(TREND); } remember(d.rows); // what the server applied: an unknown collection is no filter (but "Yours" waits for the visitor's datasets) if (d.query && !(state.coll === "mine" && !MINE_N)) state.coll = d.query.collection; + if (d.query) state.owner = d.query.owner || null; + if (d.query?.f) state.sel = Object.fromEntries(Object.entries(d.query.f).map(([k, v]) => [k, new Set(v)])); } const color = (id) => (COLL[id]?.color ? `var(--c-${COLL[id].color})` : "var(--text)"); @@ -100,17 +118,19 @@ export function skeleton() { export async function mount(el, { qs }) { root = el; state.q = qs.get("q") || ""; + state.owner = qs.get("owner")?.trim().slice(0, 200) || null; const k = { dataset: "harbor", space: "openenv", ors: "other-space" }[qs.get("k")] || qs.get("k"); // old links state.kind = KINDS.some(([x]) => x === k) ? k : "all"; state.sort = SORTS.some(([x]) => x === qs.get("s")) ? qs.get("s") : "trending"; state.coll = qs.get("c") || null; state.mine = MINE_N > 0; + state.candidates = qs.get("scope") === "all" || state.kind === "unverified-space"; state.sel = {}; (qs.get("f") || "").split(";").filter(Boolean).forEach((pair) => { const [k, v] = pair.split(":"); if (k && v) state.sel[k] = new Set(v.split("|").map(decodeURIComponent)); }); - const url = query({ trending: TREND ? 0 : TRENDING }); + const url = query({ trending: TRENDING }); const hit = cache.get(url); if (!hit || Date.now() - hit.at >= 30000) el.innerHTML = skeleton(); let d; @@ -129,8 +149,15 @@ export async function mount(el, { qs }) {

Across OpenEnv, Harbor, Verifiers and more. Open one to see what it asks and how it's graded, then run an agent on it.

+
How counts and trending work
+ +

${icon("flame", 15)}Trending

+
@@ -141,13 +168,14 @@ export async function mount(el, { qs }) { / - +
@@ -168,6 +196,7 @@ export async function mount(el, { qs }) { wire(el); show(d); writeURL(); + refreshTimer = setInterval(refreshCatalog, 45000); // the signed-in visitor's own datasets: "Yours" in Collection, their private ones among the cards if (!MINE_N) myDatasets().then((mine) => { if (root !== el) return; @@ -179,11 +208,13 @@ export async function mount(el, { qs }) { }); } -export function unmount() { document.body.classList.remove("noscroll"); ctrl?.abort(); clearTimeout(typing); root = null; } +export function unmount() { document.body.classList.remove("noscroll"); ctrl?.abort(); clearTimeout(typing); clearInterval(refreshTimer); root = null; } function writeURL() { const p = new URLSearchParams(); + if (state.candidates) p.set("scope", "all"); if (state.q) p.set("q", state.q); + if (state.owner) p.set("owner", state.owner); if (state.kind !== "all") p.set("k", state.kind); if (state.coll) p.set("c", state.coll); if (state.sort !== "trending") p.set("s", state.sort); @@ -194,35 +225,58 @@ function writeURL() { function renderStats() { if (!STATS) return; - $("#stats", root).innerHTML = `
${fmt.format(STATS.datasets)}datasets
-
${fmt.format(STATS.spaces)}environment Spaces
-
${compact.format(STATS.tasks)}tasks indexed
`; + const box = $("#stats", root), previous = box._counts || {}; + const coverage = STATS.space_task_coverage || {}; + const numbers = { datasets: STATS.datasets, spaces: STATS.spaces, tasks: STATS.dataset_tasks ?? STATS.tasks, reported: STATS.space_tasks || 0 }; + const metrics = [["datasets", "datasets"], ["spaces", state.candidates ? "Space candidates" : "checked Spaces"], ["tasks", "tasks indexed"], ["reported", "Space task entries"]]; + box.innerHTML = metrics.map(([key, label]) => `
+ ${label} + ${key === "reported" ? `reported by ${fmt.format(coverage.spaces || 0)} Spaces` : ""}
`).join(""); + for (const [key] of metrics) countUp(box.querySelector(`[data-count="${key}"]`), numbers[key], (n) => fmt.format(n), previous[key] ?? 0); + box._counts = numbers; + const at = SNAPSHOT?.built_at; + $("#catalog-freshness", root).textContent = `· updates automatically${at ? ` · catalog ${ago(time(at) / 1000)}` : ""}`; + $("#catalog-context", root).innerHTML = `

All four counts follow your current search, organization and filters. + By default, datasets and running Spaces with a recently checked environment API are shown. Offline and unverified candidates are available in Filters. + Checks run automatically; OpenEnv labels require passing OpenEnv API checks within the last hour. An API check does not certify episode or reward quality.

+

${fmt.format(numbers.tasks)} tasks indexed are dataset records loaded into this explorer's index. + ${fmt.format(numbers.reported)} Space task entries are reported by ${fmt.format(coverage.spaces || 0)} running Spaces within the last hour; individual task records are fetched when you open them. + These are entries across repositories and splits, not unique problems: forks, conversions and shared datasets can overlap. The two task counts can also overlap and are not added together. + ${coverage.partial_spaces ? `${fmt.format(coverage.partial_spaces)} Space catalogs are partial; only known split counts are included.` : ""} + Spaces without a recent task count are uncounted, not assumed empty. Space totals describe their Task API catalogs; their task bodies are loaded on demand. + ${coverage.oldest_check ? `Oldest included Space count: ${esc(new Date(coverage.oldest_check * 1000).toLocaleString())}.` : "No recent Space counts yet."}

+

Repository names, tags and an openenv.yaml filename are discovery hints only. Versions describe repository dependencies, not a confirmed installed runtime. + Automatic indexing refreshes public repositories and supported dataset indexes in batches; full task records from row datasets and Spaces are loaded on demand.

+

${esc(RANKING?.description || "Hub trending score, then total likes and monthly dataset downloads.")} ${esc(RANKING?.quality || "")}

+

Counts cover the public, tagged environment catalog and curated repositories, not a census of all Hub Spaces. + ${at ? `Catalog snapshot: . Runtime status and metrics can change.` : "Snapshot time unavailable."}

`; } // ── trending: a grid of cards, a page at a time ────────────────────────────── let trPage = 0; -function trendCard(d, i) { +function trendCard(d, i = null) { const [org, name] = d.id.split("/"); const c = COLL[d.collection]; const space = d.kind === "space"; const nums = space - ? [[icon("heart", 12), compact.format(d.likes), "likes"], ...(d.mcp ? [[icon("plug", 12), "MCP", "has MCP tools"]] : [])] + ? [[icon("heart", 12), compact.format(d.likes), "likes"], ...(d.mcp ? [[icon("plug", 12), "MCP", "tagged for MCP"]] : [])] : [[icon("download", 12), compact.format(d.downloads), "downloads last month"], [icon("heart", 12), compact.format(d.likes), "likes"], ...(d.indexed?.tasks ? [[icon("list", 12), compact.format(d.indexed.tasks), "tasks"]] : [])]; + nums.unshift([icon("flame", 12), compact.format(d.trending || 0), "Hub trending score; 0 means ordering falls back to likes and dataset downloads"]); const kind = kindTag(d, true); - return `
  • -
    ${i + 1}${kind}${c ? `${esc(c.group)}` : ""}
    -
    ${esc(org)}/${esc(name)}
    + return `
  • +
    ${i === null ? "" : `${i + 1}`}${kind}${c ? `${esc(c.group)}` : ""}
    +
    ${ownerLink(org, "/")}${esc(name)}

    ${esc(d.brief || d.heading || (c ? c.about : "") || "")}

    -
    ${nums.map(([ic, n, l]) => `${ic}${n}`).join("")}
    +
    ${d.api_status === "API checked" ? `${icon("check", 12)}API checked` : ""}${nums.map(([ic, n, l]) => `${ic}${n}`).join("")}
    ${space ? `` : ""} -
    • `; +
    `; } // the framework an environment is built for, and what it is: a Harbor dataset of tasks, or an OpenEnv Space (a server) function kindTag(d, dot = false) { if (d.kind === "space") { - const k = d.framework === "ors" ? "ors" : fw(d), name = { openenv: "OpenEnv", ors: "ORS" }[k]; - return `${icon("globe", 11)}${name ? `${name}Space` : "Space"}${dot && d.stage === "RUNNING" ? '' : ""}`; + const k = d.framework === "ors" ? "ors" : fw(d), name = { openenv: "OpenEnv", "unverified-space": "Unverified", ors: "ORS" }[k]; + return `${icon("globe", 11)}${name ? `${name}Space` : "Space"}${dot && d.stage === "RUNNING" ? '' : ""}`; } const [ic, a, b, title] = FW[d.framework] || FW.harbor; return `${icon(ic, 11)}${a}${b}`; @@ -234,19 +288,26 @@ function pageSize() { } function renderTrending() { if (!TREND) return; + $("#fineenvs-featured", root).hidden = !PROMOTED.length; + $("#promoted", root).innerHTML = PROMOTED.map((d) => trendCard(d)).join(""); + $("#featured", root).hidden = !FEATURED.length; + $("#featured", root).innerHTML = `Featured by maintainers${FEATURED.map((d) => `${esc(d.id)}`).join("")}`; + const filtered = state.owner || state.kind !== "all" || state.coll || state.q || Object.keys(state.sel).length; + $("#tr-scope", root).textContent = filtered ? "Within these filters" : "Across the catalog"; const all = storage.get("trend-view") === "all"; const top = TREND; const n = pageSize(), pages = Math.max(1, Math.ceil(top.length / n)); trPage = Math.max(0, Math.min(trPage, pages - 1)); const shown = all ? top : top.slice(trPage * n, trPage * n + n); const grid = $("#trending", root); - grid.innerHTML = shown.map((d) => trendCard(d, top.indexOf(d))).join(""); + grid.innerHTML = shown.length ? shown.map((d) => trendCard(d, top.indexOf(d))).join("") : '
  • No environments match these filters.
    • '; grid.classList.remove("turn"); void grid.offsetWidth; // replay the page-turn fade grid.classList.add("turn"); $("#tr-all", root).textContent = all ? "Show fewer" : `See all ${top.length}`; - $("#tr-pager", root).hidden = all; - $("#tr-page", root).textContent = `${trPage * n + 1}–${Math.min(top.length, trPage * n + n)} of ${top.length}`; + $("#tr-all", root).hidden = top.length <= n; + $("#tr-pager", root).hidden = all || top.length <= n; + $("#tr-page", root).textContent = top.length ? `${trPage * n + 1}–${Math.min(top.length, trPage * n + n)} of ${top.length}` : "0 results"; $("#tr-prev", root).disabled = trPage === 0; $("#tr-next", root).disabled = trPage >= pages - 1; } @@ -258,14 +319,20 @@ function quickLook(d) { const ix = d.indexed; const kinds = ix?.graded ? Object.entries(ix.graded).filter(([k]) => k !== "unknown").map(([k]) => (k === "json" ? "named scores" : "one score")) : []; const facts = d.kind === "space" ? [ - ["Status", ` ${d.stage === "RUNNING" ? "Running" : d.stage ? "Asleep" : "Unknown"}`], - ["Likes", fmt.format(d.likes)], ["Trending score", fmt.format(d.trending || 0)], - ["Kind", fw(d) === "openenv" ? `OpenEnv server${d.openenv_version ? ` ${esc(d.openenv_version)}` : ""}${d.manifest ? ` (${esc(d.manifest)})` : ""}` : d.framework === "ors" ? "ORS server" : "Docker Space"], - ["MCP tools", d.mcp ? "yes" : "not tagged"], d.hardware ? ["Hardware", esc(d.hardware)] : null, + ["Status at snapshot", esc(d.stage_label || d.stage || "Unknown")], + d.api_status ? ["OpenEnv API", `${esc(d.api_status)}${d.api_check?.checked_at ? ` · ${esc(ago(d.api_check.checked_at))}` : ""}`] : null, + d.api_mode ? ["Interface", esc(d.api_mode)] : null, + d.declared_version ? ["OpenEnv version (declared)", `${esc(d.declared_version)} · ${esc(d.version_source)}`] : null, + d.api_check?.version?.path ? ["Version source", `${esc(d.api_check.version.path)}`] : null, + d.tools_status ? ["MCP tool discovery", esc(d.tools_status)] : null, + ["Likes", fmt.format(d.likes)], ["Hub trending score", fmt.format(d.trending || 0)], + ["Kind", fw(d) === "openenv" ? `API-checked OpenEnv${d.openenv_version ? ` ${esc(d.openenv_version)}` : ""}${d.manifest ? ` (${esc(d.manifest)})` : ""}` : fw(d) === "unverified-space" ? "Unverified Space candidate; metadata is not API verification" : d.framework === "ors" ? "ORS declared" : "Docker Space"], + d.evidence ? ["OpenEnv evidence", esc(d.evidence)] : null, + ["MCP metadata", d.mcp ? "tagged; inspect live tools when opened" : "not tagged"], d.hardware ? ["Hardware", esc(d.hardware)] : null, d.updated ? ["Updated", ago(time(d.updated) / 1000)] : null, ] : [ ["Tasks", ix?.tasks != null ? fmt.format(ix.tasks) : "not indexed yet"], ["Downloads", `${compact.format(d.downloads)} last month`], - ["Likes", fmt.format(d.likes)], ["Trending score", fmt.format(d.trending || 0)], + ["Likes", fmt.format(d.likes)], ["Hub trending score", fmt.format(d.trending || 0)], kinds.length ? ["Graded by", esc(kinds.join(", "))] : null, ix?.tasks ? ["Prebuilt image", `${Math.round((100 * (ix.image || 0)) / ix.tasks)}% of tasks`] : null, d.rollouts ? ["Public rollouts", fmt.format(d.rollouts)] : null, d.updated ? ["Updated", ago(time(d.updated) / 1000)] : null, ]; @@ -273,7 +340,7 @@ function quickLook(d) { ${d.kind === "space" ? `` : ""}
    ${c ? `${icon(c.icon, 12)}${esc(c.group)}` : ""} ${icon(d.kind === "space" ? "globe" : "database", 11)}${d.kind === "space" ? "Space" : "Dataset"}${d.badges.map((b) => `${esc(b)}`).join("")}
    -

    ${esc(org)}/${esc(name)}

    ${d.heading && d.heading !== name ? `

    ${esc(d.heading)}

    ` : ""}
    +

    ${ownerLink(org, "/")}${esc(name)}

    ${d.heading && d.heading !== name ? `

    ${esc(d.heading)}

    ` : ""}
    ${d.brief ? `

    ${esc(d.brief)}

    ` : ""}
    ${facts.filter(Boolean).map(([k, v]) => `
    ${k}
    ${v}
    `).join("")}
    @@ -287,7 +354,7 @@ function quickLook(d) { // ── asking the server ──────────────────────────────────────────────────────── // Any change of search, kind, collection, facet or order: the controls answer at once, the cards and counts when the // server does. A newer question cancels an older one still on its way; a slow answer shows placeholder cards. -function run({ debounce = 0, trending = false } = {}) { +function run({ debounce = 0, trending = true } = {}) { writeURL(); renderFacets(); renderActive(); clearTimeout(typing); @@ -303,7 +370,7 @@ async function ask(withTrending) { if (my !== seq || !root) return; apply(d); show(d); - if (d.trending) { renderStats(); renderTrending(); } + if (d.trending) { trPage = 0; renderStats(); renderTrending(); } writeURL(); } catch (e) { if (e.name === "AbortError" || my !== seq || !root) return; @@ -311,6 +378,26 @@ async function ask(withTrending) { toast(`Couldn't load the environments: ${e.message}`); } finally { clearTimeout(slow); } } + +async function refreshCatalog() { + // Refresh an idle first page. Never replace expanded results or a focused + // card/filter while someone is browsing, and never race a newer query. + if (!root || document.hidden || refreshing || page !== 1 || Date.now() - lastAnswer < 40000) return; + const active = document.activeElement; + if (active?.closest(".results, .filters, .fe-trend, .modal")) return; + const el = root, version = seq, url = query({ trending: TRENDING }); + refreshing = true; + try { + const d = await get(url); + if (root !== el || seq !== version || page !== 1 || url !== query({ trending: TRENDING })) return; + apply(d); + show(d); + renderStats(); + renderTrending(); + } catch { /* Keep the last successful catalog; retry on the next tick. */ } + finally { refreshing = false; } +} + function loading(on) { const list = $("#list", root); $(".results", root)?.setAttribute("aria-busy", on ? "true" : "false"); @@ -321,6 +408,7 @@ function loading(on) { list.innerHTML = sk.cards(6); } function show(d) { + lastAnswer = Date.now(); res = d; rows = d.rows.slice(); page = 1; @@ -354,6 +442,8 @@ async function more() { const phone = () => matchMedia("(max-width: 760px)").matches; function renderFacets() { if (!res?.facets || !root) return; + const active = document.activeElement; + const focus = active?.closest('#facets') && active.matches('button') ? { ...active.dataset } : null; const F = res.facets; const collFacet = (() => { const counts = F.collection || {}; @@ -368,22 +458,27 @@ function renderFacets() { return ``; }).join("")}`; $("#facets", root).innerHTML = kindFacet + collFacet + FACETS.map(([key, label]) => { const counts = new Map(F[key] || []); + if (key === "health" && counts.size && !counts.has("API checked")) counts.set("API checked", 0); + if (key === "tools" && counts.size && !counts.has("Tools discovered")) counts.set("Tools discovered", 0); const sel = state.sel[key] || new Set(); sel.forEach((v) => { if (!counts.has(v)) counts.set(v, 0); }); - if (counts.size <= 1 && !sel.size) return ""; + if (counts.size <= 1 && !sel.size && !(["health", "oe", "tools", "mode"].includes(key) && counts.size)) return ""; let items = [...counts.entries()].sort(key === "size" ? (a, b) => SIZE_ORDER.indexOf(a[0]) - SIZE_ORDER.indexOf(b[0]) + : key === "oe" ? (a, b) => versionGroup(a[0]) - versionGroup(b[0]) || String(b[0]).localeCompare(String(a[0]), undefined, { numeric: true }) : (a, b) => b[1] - a[1] || String(a[0]).localeCompare(String(b[0]))); const limit = expanded.has(key) ? Infinity : (phone() ? 5 : 8); const hidden = Math.max(0, items.length - limit); items = items.slice(0, limit); const open = facetOpen.has(key) ? facetOpen.get(key) : (!phone() || sel.size > 0); return `

    ${esc(label)}

    ${sel.size ? `${sel.size} selected` : ""}${icon("chevronDown", 14, "chev")}     + ${FACET_HELP[key] ? `

    ${esc(FACET_HELP[key])}

    ` : ""} ${items.map(([v, c]) => ``).join("")} ${hidden || expanded.has(key) ? `` : ""}
    `; }).join(""); const nf = Object.values(state.sel).reduce((n, s) => n + s.size, 0); $("#nfilt", root).textContent = nf ? ` (${nf})` : ""; + if (focus) [...$("#facets", root).querySelectorAll('button')].find((b) => Object.entries(focus).every(([k, v]) => b.dataset[k] === v))?.focus({ preventScroll: true }); } function toggle(key, v) { const set = (state.sel[key] ||= new Set()); @@ -395,27 +490,40 @@ function setColl(id) { state.coll = id || null; run(); } function renderActive() { if (!res || !root) return; const labels = Object.fromEntries(FACETS); + $("#include-candidates", root).checked = state.candidates; const pills = []; + const heading = $(".ex-head h1", root), description = $(".ex-head p", root); + heading.textContent = state.owner ? `Environments by ${state.owner}` : "Explore RL environments on Hugging Face"; + description.textContent = state.owner ? "Datasets and Spaces from this owner in the explorer catalog. Search or filter to find an environment." : "Across OpenEnv, Harbor, Verifiers and more. Open one to see what it asks and how it's graded, then run an agent on it."; + $("#q", root).placeholder = state.owner ? `Search ${state.owner} environments…` : "Search environments, owners, tags…"; + setMeta({ title: state.owner ? `${state.owner} environments` : undefined }); + if (state.owner) pills.push(``); if (state.kind !== "all") pills.push(``); if (state.coll) pills.push(``); Object.entries(state.sel).forEach(([k, set]) => set.forEach((v) => pills.push(``))); $("#active", root).innerHTML = pills.join(""); - const noun = state.kind === "openenv" ? "Space" : state.kind !== "all" ? "dataset" : "environment"; + const noun = ["openenv", "unverified-space", "other-space"].includes(state.kind) ? "Space" : state.kind !== "all" ? "dataset" : "environment"; $("#count", root).textContent = `${fmt.format(res.total)} ${noun}${res.total === 1 ? "" : "s"}`; const sr = $("#show-results", root); if (sr) sr.textContent = `Show ${fmt.format(res.total)} ${noun}${res.total === 1 ? "" : "s"}`; } // ── list ───────────────────────────────────────────────────────────────────── +function checkTitle(d) { + const when = d.api_check?.checked_at ? new Date(d.api_check.checked_at * 1000).toLocaleString() : "Never checked"; + return `${when}. API checks only; episode execution and rewards are not validated.`; +} function card(d) { const [org, name] = d.id.split("/"); const c = COLL[d.collection]; const chips = d.kind === "space" ? [ - `${d.stage === "RUNNING" ? "Running" : d.stage ? "Asleep" : "Unknown"}`, + d.api_status ? `${icon(d.api_status === "API checked" ? "check" : "info", 11)}${esc(d.api_status)}${d.api_check?.checked_at ? ` · ${esc(ago(d.api_check.checked_at))}` : ""}` : "", + `${esc(d.stage_label || d.stage || "Unknown")}`, d.likes ? `${compact.format(d.likes)} likes` : "", - d.mcp ? `${icon("plug", 11)}MCP tools` : "", - d.openenv ? `OpenEnv${d.openenv_version ? ` ${esc(d.openenv_version)}` : ""}` : "", + d.mcp ? `${icon("plug", 11)}MCP tagged` : "", + d.declared_version && d.declared_version !== "Unknown" ? `Declared OpenEnv ${esc(d.declared_version)}` : "", + d.evidence ? `${esc(d.evidence)}` : "", ] : [ d.indexed?.tasks ? `${fmt.format(d.indexed.tasks)} tasks` : "", `${compact.format(d.downloads)} downloads`, @@ -427,15 +535,15 @@ function card(d) { // a search that matched a tag the card doesn't otherwise show: show that tag, so the match makes sense const q = state.q.toLowerCase(); if (q) (d.tags || []).filter((t) => t.toLowerCase().includes(q)).slice(0, 2).forEach((t) => chips.push(`${icon("search", 11)}${esc(t)}`)); - return `
  • + return `
  • ${d.kind === "space" ? `` : ""}
    ${kindTag(d)} - ${c ? `${icon(c.icon, 13)}${esc(c.group)}/` : ""}${esc(org)} + ${c ? `${icon(c.icon, 13)}${esc(c.group)}/` : ""}${ownerLink(org)} ${d.pinned ? `${icon("flag", 11)}Pinned` : ""}${d.updated ? `${esc(ago(time(d.updated) / 1000))}` : ""}
    -

    ${esc(name)}

    +

    ${esc(name)}

    ${d.brief || d.heading ? `

    ${esc(d.brief || d.heading)}

    ` : ""} -
    ${chips.filter(Boolean).join("")}
    • `; +
    ${chips.filter(Boolean).join("")}
    `; } function renderList() { const list = $("#list", root); @@ -452,6 +560,11 @@ function moreRow() { } function wire(el) { + el.addEventListener("click", (e) => { + const card = e.target.closest("[data-open]"); + if (!card || e.defaultPrevented || e.target.closest("a,button,input,select") || e.button || e.ctrlKey || e.metaKey || e.shiftKey || e.altKey || window.getSelection()?.toString()) return; + nav(card.dataset.open); + }); $("#q", el).addEventListener("input", (ev) => { state.q = ev.target.value.trim(); run({ debounce: 160 }); }); $("#sort", el).addEventListener("change", (ev) => { state.sort = ev.target.value; run(); }); $("#facets", el).addEventListener("click", (ev) => { const b = ev.target.closest("[data-kind]"); if (b) { state.kind = b.dataset.kind; state.coll = null; run(); } }); @@ -479,25 +592,51 @@ function wire(el) { $("#active", el).addEventListener("click", (ev) => { const p = ev.target.closest(".pill"); if (!p) return; - if (p.dataset.kind) { state.kind = "all"; run(); } else if (p.dataset.coll != null) setColl(null); else toggle(p.dataset.k, p.dataset.v); + if (p.hasAttribute("data-clear-owner")) { state.owner = null; run(); } + else if (p.dataset.kind) { state.kind = "all"; run(); } else if (p.dataset.coll != null) setColl(null); else toggle(p.dataset.k, p.dataset.v); + }); + const clearAll = () => { state.owner = null; state.candidates = false; state.sel = {}; state.q = ""; $("#q", el).value = ""; state.coll = null; state.kind = "all"; run(); }; + $("#include-candidates", el).addEventListener("change", (e) => { + state.candidates = e.target.checked; + if (!state.candidates && state.kind === "unverified-space") state.kind = "all"; + run(); }); - const clearAll = () => { state.sel = {}; state.q = ""; $("#q", el).value = ""; state.coll = null; state.kind = "all"; run(); }; $("#clear", el).addEventListener("click", clearAll); $("#clear2", el).addEventListener("click", clearAll); - const sheet = (on) => { $("#filters", el).classList.toggle("open", on); $("#sheet-scrim", el).hidden = !on; document.body.classList.toggle("noscroll", on); }; + const sheet = (on) => { + $("#filters", el).classList.toggle("open", on); + syncFilters(); + $(on ? "#close-filters" : "#open-filters", el).focus({ preventScroll: true }); + }; + syncFilters(); + el.addEventListener("keydown", (e) => { + const drawer = $("#filters", el); + if (!phone() || !drawer.classList.contains("open")) return; + if (e.key === "Escape") { e.preventDefault(); sheet(false); } + if (e.key === "Tab") { + const items = [...drawer.querySelectorAll('button:not(:disabled), summary, input, select, a[href]')].filter((x) => x.getClientRects().length); + const first = items[0], last = items.at(-1); + if (e.shiftKey && (document.activeElement === first || !drawer.contains(document.activeElement))) { e.preventDefault(); last?.focus(); } + else if (!e.shiftKey && (document.activeElement === last || !drawer.contains(document.activeElement))) { e.preventDefault(); first?.focus(); } + } + }); $("#open-filters", el).addEventListener("click", () => sheet(true)); $("#close-filters", el).addEventListener("click", () => sheet(false)); $("#sheet-scrim", el).addEventListener("click", () => sheet(false)); $("#show-results", el).addEventListener("click", () => { sheet(false); $("#toolbar", el).scrollIntoView({ block: "start" }); }); - $("#clear3", el).addEventListener("click", () => { state.sel = {}; state.coll = null; run(); }); + $("#clear3", el).addEventListener("click", () => { state.sel = {}; state.coll = null; state.candidates = false; state.kind = "all"; run(); }); $("#more", el).addEventListener("click", more); $("#random", el).addEventListener("click", async () => { // a task from the chosen collection's indexed datasets, each equally likely; else the server picks from all of them try { let pick = ""; - if (state.coll && COLL[state.coll] && state.coll !== "mine") { - const d = await get(`/api/search?collection=${encodeURIComponent(state.coll)}&size=100&facets=0`); + if (state.owner || (state.coll && COLL[state.coll] && state.coll !== "mine")) { + const p = new URLSearchParams({ size: "100", facets: "0" }); + if (state.owner) p.set("owner", state.owner); + if (state.coll) p.set("collection", state.coll); + const d = await get(`/api/search?${p}`); const pool = d.rows.filter((x) => x.kind === "dataset" && x.indexed?.tasks); + if (!pool.length && state.owner) { toast("No indexed dataset tasks are available for this owner yet."); return; } pick = pool.length ? pool[Math.floor(Math.random() * pool.length)].id : ""; } const r = await api(`/api/random${pick ? `?d=${encodeURIComponent(pick)}` : ""}`); @@ -507,5 +646,20 @@ function wire(el) { if (phone()) $("#q", el).placeholder = "Search environments…"; } +function syncFilters() { + const drawer = root && $("#filters", root); + if (!drawer) return; + const mobile = phone(); + if (!mobile) drawer.classList.remove("open"); + const open = mobile && drawer.classList.contains("open"); + drawer.inert = mobile && !open; + if (mobile && !open) drawer.setAttribute("aria-hidden", "true"); else drawer.removeAttribute("aria-hidden"); + if (open) { drawer.setAttribute("role", "dialog"); drawer.setAttribute("aria-modal", "true"); } + else { drawer.removeAttribute("role"); drawer.removeAttribute("aria-modal"); } + $("#open-filters", root).setAttribute("aria-expanded", String(open)); + $("#sheet-scrim", root).hidden = !open; + document.body.classList.toggle("noscroll", open); +} + let resizeT; // the page size follows the width: re-lay the trending grid when it changes -addEventListener("resize", () => { clearTimeout(resizeT); resizeT = setTimeout(() => { if (root && TREND && $("#trending", root)) renderTrending(); }, 150); }); +addEventListener("resize", () => { clearTimeout(resizeT); resizeT = setTimeout(() => { syncFilters(); if (root && TREND && $("#trending", root)) renderTrending(); }, 150); }); diff --git a/web/js/main.js b/web/js/main.js index 1d15844d7e1756b9cab80f9455573afa5a0a3310..2cf824eb0f04e93283ec6902bf5b03dee2012300 100644 --- a/web/js/main.js +++ b/web/js/main.js @@ -11,7 +11,7 @@ // /community everyone's public rollouts, by model // Older addresses still open: #/… (the hash router this app had), and the MiMo RL Environment Explorer's #/task/, // #/compare/ and #/rewards. -import { $, api, esc, storage, closeModal, openDialog, progress, spinner, toast, nav, setMeta } from "./util.js"; +import { $, api, esc, storage, closeModal, openDialog, progress, spinner, toast, nav, setMeta, emptyState } from "./util.js"; import { icon } from "./icons.js"; import { getSession, setSession, refreshActive } from "./session.js"; import { wireTopSearch } from "./topsearch.js"; @@ -24,6 +24,8 @@ const VIEWS = { home: () => import("./home.js"), dataset: () => import("./env.js const MIMO = "XiaomiMiMo/MiMo-V2.6-RL-oss"; const framed = window.top !== window.self; +const decodePath = (s) => { try { return decodeURIComponent(s); } catch { return s; } }; + function parse(url) { const [path, qs] = url.split("?"); let m = path.match(/^\/run\/([\w-]+)$/); @@ -31,9 +33,9 @@ function parse(url) { if (/^\/runs\/?$/.test(path)) return { name: "runs", arg: {}, qs }; if (/^\/community\/?$/.test(path)) return { name: "community", arg: {}, qs }; m = path.match(/^\/s\/([^/]+)\/([^/]+)\/?$/); - if (m) return { name: "space", arg: { spec: `${decodeURIComponent(m[1])}/${decodeURIComponent(m[2])}` }, qs }; + if (m) return { name: "space", arg: { spec: `${decodePath(m[1])}/${decodePath(m[2])}` }, qs }; m = path.match(/^\/compare\/([^/]+)\/([^/]+)\/(.+)$/); - if (m) return { name: "compare", arg: { spec: `${decodeURIComponent(m[1])}/${decodeURIComponent(m[2])}`, ref: decodeURIComponent(m[3]) }, qs }; + if (m) return { name: "compare", arg: { spec: `${decodePath(m[1])}/${decodePath(m[2])}`, ref: decodePath(m[3]) }, qs }; m = path.match(/^\/(task|compare)\/([^/]+)$/); // the MiMo explorer's addresses if (m) return { redirect: `/${m[1] === "task" ? "t" : "compare"}/${MIMO}/${m[2]}${qs ? `?${qs}` : ""}` }; if (/^\/rewards\/?$/.test(path)) return { redirect: `/d/${MIMO}?rewards=1` }; @@ -44,9 +46,9 @@ function parse(url) { return { redirect: `/t/${m[1]}/${m[2]}/${c === "default" ? "" : `${encodeURIComponent(c)}/`}${encodeURIComponent(sp)}/${i}${q.toString() ? `?${q}` : ""}` }; } m = path.match(/^\/d\/([^/]+)\/([^/]+)\/?$/); - if (m) return { name: "dataset", arg: { spec: `${decodeURIComponent(m[1])}/${decodeURIComponent(m[2])}` }, qs }; + if (m) return { name: "dataset", arg: { spec: `${decodePath(m[1])}/${decodePath(m[2])}` }, qs }; m = path.match(/^\/t\/([^/]+)\/([^/]+)\/?(.*)$/); - if (m) return { name: "task", arg: { spec: `${decodeURIComponent(m[1])}/${decodeURIComponent(m[2])}`, path: decodeURIComponent(m[3] || "") }, qs }; + if (m) return { name: "task", arg: { spec: `${decodePath(m[1])}/${decodePath(m[2])}`, path: decodePath(m[3] || "") }, qs }; return { name: "home", arg: {}, qs }; } @@ -74,6 +76,13 @@ async function route() { const old = $("#view"), view = old.cloneNode(false); old.replaceWith(view); await mod.mount(view, { ...arg, qs: qs ? new URLSearchParams(qs) : new URLSearchParams() }); + } catch (error) { + if (token !== routing) return; + current?.unmount?.(); + current = null; + $("#view").innerHTML = `
    ${emptyState("alert", "Couldn't load this page", esc(error.message || "Please try again."), + 'Explore environments')}
    `; + $("#page-retry").addEventListener("click", route); } finally { progress.done(); } } diff --git a/web/js/render.js b/web/js/render.js index 6d850d52e60be7299be226ada90642b3d172bc05..784c96863a83caaa4ba36ed0fe5ff285c5dfca89 100644 --- a/web/js/render.js +++ b/web/js/render.js @@ -11,6 +11,11 @@ const AUDIO_URL = /^https:\/\/[^\s"'<>]+\.(wav|mp3|ogg|flac|m4a)(\?[^\s"'<>]*)?$ function mediaOf(key, v, ctx) { if (typeof v !== "string" || v.length < 16) return null; + if (key === "media_url" && /^https:\/\/[^\s"'<>]+$/.test(v)) { + if (/^audio\/[\w.+-]+$/.test(ctx.mime || "")) return ``; + if (/^image\/(png|jpeg|gif|webp|avif)$/.test(ctx.mime || "")) return `Task image`; + if (/^video\/(mp4|webm|ogg)$/.test(ctx.mime || "")) return ``; + } if (v.startsWith("data:image/") && /^data:image\/(png|jpe?g|gif|webp);base64,/.test(v)) return `${esc(key)}`; if (v.startsWith("data:audio/") && /^data:audio\/[\w.+-]+;base64,/.test(v)) return ``; if (IMG_URL.test(v)) return `${esc(key)}`; @@ -60,7 +65,7 @@ function table(rows, depth) { `${keys.map((k) => `${scalar(r[k]) ? cell(r[k]) : value(r[k], k, depth + 1)}`).join("")}`).join("")} ${rows.length > 100 ? `

    ${fmt.format(rows.length - 100)} more rows

    ` : ""}`; } -const cell = (x) => (x == null ? `–` : typeof x === "string" ? esc(x.length > 140 ? x.slice(0, 140) + "…" : x) : `${esc(String(x))}`); +const cell = (x) => (x == null ? `–` : typeof x === "string" ? `${esc(x.length > 140 ? x.slice(0, 140) + "…" : x)}` : `${esc(String(x))}`); export function value(v, key = "", depth = 0, ctx = {}) { if (depth > 7) return `…`; diff --git a/web/js/reward-stats.js b/web/js/reward-stats.js new file mode 100644 index 0000000000000000000000000000000000000000..f0030d0f7ecca06ee94a3791ea8cd7cd17cd744c --- /dev/null +++ b/web/js/reward-stats.js @@ -0,0 +1,9 @@ +// A reward's scale belongs to its environment. Retain negative and unbounded scores. +export function rewardDistribution(values, bins = 10) { + const xs = values.filter(Number.isFinite); + const min = xs.reduce((a, x) => Math.min(a, x), 0); + const max = xs.reduce((a, x) => Math.max(a, x), 1); + const hist = Array(bins).fill(0); + for (const x of xs) hist[Math.max(0, Math.min(bins - 1, Math.floor((x - min) / (max - min) * bins)))]++; + return { hist, min, max, width: (max - min) / bins }; +} diff --git a/web/js/run-mimo.js b/web/js/run-mimo.js index 3c92cc7043668f50a4ca322ff58e33d0a23528c7..dd18be25d00c4261361a0141ac62565458fc6432 100644 --- a/web/js/run-mimo.js +++ b/web/js/run-mimo.js @@ -141,7 +141,7 @@ function renderHead(el, st) { ? `` : ""; $("#rp-head", el).innerHTML = ` -
    ${icon(DOMAIN_ICON[r.domain], 13)}${esc(DOMAIN_NAME[r.domain])}${statusPill(r.status)}${visibilityBadge(r.visibility)} +
    ${icon(DOMAIN_ICON[r.domain] || "cpu", 13)}${esc(DOMAIN_NAME[r.domain] || (r.runner === "nemo-gym" ? "NeMo Gym" : r.domain))}${statusPill(r.status)}${visibilityBadge(r.visibility)} ${owner ? "" : 'shared anonymously'} ${icon("clock", 13)}${dur(end - start)} · started ${ago(r.created_at)}

    ${esc(r.title)}

    @@ -149,7 +149,7 @@ function renderHead(el, st) { ${icon("cpu", 14)}${esc(r.endpoint ? r.model : r.model.split("/")[1] || r.model)} via ${r.endpoint ? `your endpoint ${esc(r.endpoint.host)}` : esc(r.provider || "auto")} ${paramsText(r.params) ? `${icon("gauge", 14)}${esc(paramsText(r.params))}` : ""} ${r.judge ? `${icon("scale", 14)}judge ${esc(r.judge.split("/")[1] || r.judge)}` : ""} - ${r.domain === "music" ? `${icon("message", 14)}single model call` : `${icon("terminal", 14)}OpenCode on the MiMo harness`}${r.flavor ? `${icon("box", 14)}${esc(r.flavor)}` : ""}
    + ${r.runner === "nemo-gym" ? `${icon("check", 14)}One prediction · NeMo Gym verifier` : r.domain === "music" ? `${icon("message", 14)}single model call` : `${icon("terminal", 14)}OpenCode on the MiMo harness`}${r.flavor ? `${icon("box", 14)}${esc(r.flavor)}` : ""}
    ${icon("flag", 13)}Report${vis} ${icon("columns", 13)}Compare ${icon("file", 13)}View task${action}
    @@ -160,7 +160,7 @@ function renderPhases(el, st) { const seen = {}; st.events.filter((e) => e.kind === "phase").forEach((e) => { (seen[e.name] ||= {})[e.status] = e; }); const failed = ["failed", "cancelled", "interrupted"].includes(st.run.status); - const noSandbox = st.run.domain === "music"; + const noSandbox = st.run.domain === "music" || st.run.runner === "nemo-gym"; const list = PHASES.filter(([k]) => !(noSandbox && (k === "sandbox" || k === "setup"))); const box = $("#rp-phases", el); box.style.setProperty("--n", list.length); @@ -208,8 +208,8 @@ function item(e, st) { const t = dur(e.t); const summary = (inner) => `${icon("chevronRight", 13, "chev")}${inner}`; switch (e.kind) { - case "prompt": return row("ev-prompt", "doc", t, `
    ${summary(`Instructions to the model${(e.text || "").length.toLocaleString()} characters`)} -
    ${esc(e.text)}

    ${e.harness === "opencode" + case "prompt": return row("ev-prompt", "doc", t, `

    ${summary(`Instructions to the model${(e.text || "").length.toLocaleString()} characters`)} +
    ${esc(e.text)}

    ${e.harness === "nemo-gym" || st.run.runner === "nemo-gym" ? "The native Responses request: conversation history and function definitions. The reference action is withheld." : e.harness === "opencode" ? "The task's instructions, sent as the first user message. OpenCode adds its own system prompt and tool definitions." : "The whole request: one user message, no system prompt."}

    `); case "text": return row("ev-text", "message", t, `
    ${md(e.text)}
    `); @@ -253,14 +253,15 @@ export function diffHtml(text) { } function renderGrade(el, st) { - const g = [...st.events].reverse().find((e) => e.kind === "checks"); + const g = [...st.events].reverse().find((e) => e.kind === "checks") || + (st.run.status === "done" && Number.isFinite(st.run.reward) ? { reward: st.run.reward, summary: "Recorded reward. Detailed grading events are unavailable." } : null); const box = $("#rp-grade", el); const h = `

    ${icon("scale", 14)}Grade

    `; if (!g) { box.innerHTML = `${h}

    ${isLive(st.run) ? "Appears here once the agent finishes and the task's grader has run." : "This rollout was not graded."}

    `; return; } - const rw = g.reward; + const rw = g.reward ?? st.run.reward; const img = [...st.events].reverse().find((e) => e.kind === "image"); const src = img && `/api/runs/${encodeURIComponent(st.id)}/artifacts/${encodeURIComponent(img.name)}`; const long = (g.summary || "").length > 180; @@ -272,8 +273,7 @@ function renderGrade(el, st) { const wsum = rubric ? g.checks.reduce((n, c) => n + wt(c), 0) || 1 : 1; box.innerHTML = `${h}
    ${src ? `What the judge saw` : ""} -
    ${rewardText(rw)}reward, out of 1
    - ${rw != null ? `
    ` : ""} +
    ${rewardText(rw)}${esc(st.run.reward_key || "reward")}
    ${g.formula ? `

    ${esc(g.formula)}

    ` : ""} ${long || !g.summary ? "" : `

    ${esc(g.summary)}

    `} ${(g.checks || []).length ? `
      ${g.checks.map((c) => `
    • @@ -329,8 +329,9 @@ function renderMeta(el, st) { ${row("Judge", r.judge ? code(r.judge) : "")} ${row("Thinking", esc(think))} ${row("Temperature", p.temperature != null ? esc(p.temperature) : "model default")} - ${r.domain === "music" ? row("Max tokens", esc(p.max_tokens || p.max_tokens_used || 100000)) : row("Step cap", esc(p.steps || "task default")) + row("Time limit", p.timeout_min ? esc(p.timeout_min + " min") : "task default") + row("Max tokens / step", esc(p.max_tokens || "default"))} - ${row("Harness", r.domain === "music" ? "single model call" : `OpenCode ${esc(pv.harness?.installed || pv.harness?.version || "")}`)} + ${r.domain === "music" || r.runner === "nemo-gym" ? row("Max tokens", esc(p.max_tokens || p.max_tokens_used || 100000)) : row("Step cap", esc(p.steps || "task default")) + row("Time limit", p.timeout_min ? esc(p.timeout_min + " min") : "task default") + row("Max tokens / step", esc(p.max_tokens || "default"))} + ${row("Harness", r.runner === "nemo-gym" ? "Single prediction; tool calls are not executed" : r.domain === "music" ? "single model call" : `OpenCode ${esc(pv.harness?.installed || pv.harness?.version || "")}`)} + ${row("Verifier revision", r.runner === "nemo-gym" ? code(pv.commit || "unknown") : "")} ${row("Sandbox", r.flavor ? esc(r.flavor) : "")} ${row("Image", r.image ? code(r.image.replace("docker.io/", "")) : "")} ${row("Explorer", pv.app ? `v${esc(pv.app.version)} · ${code(pv.app.source)}` : "")} diff --git a/web/js/run.js b/web/js/run.js index fa145bcf342ea596e3c77a70cb7d6f23e91c216d..8f0c23f836a01c340f871f58d3743bdc967825e7 100644 --- a/web/js/run.js +++ b/web/js/run.js @@ -40,7 +40,7 @@ export async function mount(el, { id }) { return; } if (!alive) return; - if (first.run.runner === "mimo") { + if (["mimo", "nemo-gym"].includes(first.run.runner)) { other = await import("./run-mimo.js"); if (alive) other.mount(el, id, first); return; @@ -200,8 +200,7 @@ function renderGrade(el, st) { const named = Object.entries(r.rewards || {}); box.innerHTML = `${h}
      ${rewardText(r.reward)}reward, from the task's ${r.judge ? `grader, with ${esc(r.judge)} as its judge${r.judge_calls != null ? ` (${r.judge_calls} call${r.judge_calls === 1 ? "" : "s"})` : ""}` : "tests"}
      -
      - ${named.length > 1 ? `
        ${named.map(([k, v]) => `
      • 0 ? "na" : "bad"}">${icon(v >= 0.999 ? "check" : v > 0 ? "more" : "x", 14)} + ${named.length > 1 ? `
          ${named.map(([k, v]) => `
        • ${icon("more", 14)}
          ${esc(k)}
          ${Number(v).toFixed(2)}
          • `).join("")}
        ` : ""} ${(r.findings || []).length ? `
        ${icon("chevronRight", 14, "chev")}Capture notes (${r.findings.length})
          ${r.findings.map((f) => `
        • ${esc(f)}
          • `).join("")}
        ` : ""} diff --git a/web/js/runs.js b/web/js/runs.js index ae7191056687120229a48f78df33360452c24781..d2c46d0c72a6be171532d9f3b1dc2ee92aae7c65 100644 --- a/web/js/runs.js +++ b/web/js/runs.js @@ -35,9 +35,8 @@ export async function mount(el) { : `
        ${runs.length ? emptyState("filter", "Nothing in this view", "Try another filter.") : emptyState("play", "No rollouts yet", "Open a task and run one. It shows up here with its live trajectory and reward.", `${icon("grid", 15)}Browse environments`)}
        `; const scored = runs.filter((r) => r.reward != null); - const full = scored.filter((r) => r.reward >= 0.999).length; $("#rs-stats", el).innerHTML = `
        ${runs.length}rollouts
        -
        ${scored.length ? `${full}/${scored.length}` : "–"}full marks
        `; +
        ${scored.length}scored rollouts
        `; }; const load = async () => { if (!alive) return; @@ -57,6 +56,6 @@ function row(r) { ${esc(r.title)}${icon(COLL[r.collection] || "database", 12)}${esc(r.dataset)} ${esc(model)} ${rewardBadge(r.reward, r.status)} - ${esc(r.runner === "mimo" ? (r.domain === "music" ? "MiMo scorer" : "MiMo harness") : r.harness)} + ${esc(r.runner === "nemo-gym" ? "NeMo Gym prediction" : r.runner === "mimo" ? (r.domain === "music" ? "MiMo scorer" : "MiMo harness") : r.harness)} ${ago(r.created_at)}${dur(end - (r.started_at || r.created_at))}`; } diff --git a/web/js/schemaform.js b/web/js/schemaform.js index 6e58aa42411d283f7c62d9c020911bbc83f20a48..fd356e4a3f3ea954bc45da4c6a2619e253250e45 100644 --- a/web/js/schemaform.js +++ b/web/js/schemaform.js @@ -1,78 +1,154 @@ -// A form from a JSON schema (an MCP tool's inputSchema, an OpenEnv action schema): one field per property, typed -// inputs for strings, numbers, booleans and enums, a JSON box for anything nested. read() turns it back into an object -// (fields left empty are left out, so the server's defaults apply). +// Forms preserve the JSON schema's values and defaults. Complex unions and free-form +// objects use a JSON editor instead of guessing which action variant the user meant. import { esc } from "./util.js"; -// strings that hold prose or code get a box; the rest (a word, an id, a path) a line const LONG = /(prompt|text|code|content|description|instruction|message|answer|transcript|latex|query|body|script|reasoning|patch)/i; +const own = (o, k) => Object.prototype.hasOwnProperty.call(o || {}, k); +const object = (v) => v !== null && typeof v === "object" && !Array.isArray(v); -function resolve(p, defs) { - if (p?.$ref) return defs[p.$ref.split("/").pop()] || {}; - return p || {}; +export function resolveSchema(schema, root = schema, seen = new Set()) { + if (!object(schema)) return {}; + if (!schema.$ref) return schema; + const ref = schema.$ref; + if (typeof ref !== "string" || !ref.startsWith("#/") || seen.has(ref) || seen.size > 20) return {}; + let target = root; + for (const part of ref.slice(2).split("/").map((s) => s.replace(/~1/g, "/").replace(/~0/g, "~"))) { + if (!own(target, part)) return {}; + target = target[part]; + } + const { $ref, ...rest } = schema; + return { ...resolveSchema(target, root, new Set([...seen, ref])), ...rest }; } -// a property's type, looking through anyOf/oneOf with null (Optional[...]) -function kind(p, defs) { - p = resolve(p, defs); - if (p.enum) return { t: "enum", p }; - if (p.const !== undefined) return { t: "const", p }; - const alts = (p.anyOf || p.oneOf || []).map((x) => resolve(x, defs)).filter((x) => x.type !== "null"); - if (alts.length === 1) return { ...kind({ title: p.title, ...alts[0] }, defs), nullable: true }; - if (alts.length > 1) return { t: "json", p }; - const t = Array.isArray(p.type) ? p.type.find((x) => x !== "null") : p.type; - if (t === "string") return { t: p.maxLength > 200 || (p.maxLength == null && !p.format && !p.enum && LONG.test(p.title || "")) ? "text" : "string", p }; - if (t === "integer" || t === "number") return { t: "number", p, int: t === "integer" }; - if (t === "boolean") return { t: "bool", p }; - return { t: "json", p }; +function kind(raw, root) { + const p = resolveSchema(raw, root); + if (own(p, "const")) return { t: "const", p }; + if (Array.isArray(p.enum)) return { t: "enum", p }; + const alternatives = p.anyOf || p.oneOf; + if (Array.isArray(alternatives)) { + const choices = alternatives.map((x) => resolveSchema(x, root)).filter((x) => x.type !== "null"); + if (choices.length === 1 && choices.length < alternatives.length) { + const { anyOf, oneOf, ...rest } = p; + return { ...kind({ ...choices[0], ...rest }, root), nullable: true }; + } + return { t: "json", p }; + } + if (p.allOf) return { t: "json", p }; + const ts = Array.isArray(p.type) ? p.type.filter((x) => x !== "null") : [p.type]; + const t = ts.length === 1 ? ts[0] : undefined; + return { t: t === "string" ? (LONG.test(p.title || "") ? "text" : "str") : + t === "integer" ? "int" : t === "number" ? "num" : t === "boolean" ? "bool" : "json", p, + nullable: Array.isArray(p.type) && p.type.includes("null") }; } -const label = (k, p) => esc(p.title || k); +// Deliberately a bounded, conservative subset of JSON Schema: server validation +// remains authoritative for formats, patterns and custom framework constraints. +export function validateValue(value, schema, path = "Input", root = schema, depth = 0) { + if (depth > 24) return; + if (schema === false) throw new Error(`${path} is not allowed`); + const p = resolveSchema(schema, root); + const equal = (a, b) => JSON.stringify(a) === JSON.stringify(b); + if (own(p, "const") && !equal(value, p.const)) throw new Error(`${path} must be ${JSON.stringify(p.const)}`); + if (p.enum && !p.enum.some((x) => equal(x, value))) throw new Error(`${path} must be one of the listed choices`); + for (const option of p.allOf || []) validateValue(value, option, path, root, depth + 1); + const alternatives = p.anyOf || p.oneOf; + if (alternatives) { + const matches = alternatives.filter((x) => { try { validateValue(value, x, path, root, depth + 1); return true; } catch { return false; } }); + if (!matches.length || (p.oneOf && matches.length !== 1)) throw new Error(`${path} must match ${p.oneOf ? "exactly one" : "one"} of the allowed schemas`); + } + const types = Array.isArray(p.type) ? p.type : p.type ? [p.type] : []; + const is = (t) => t === "null" ? value === null : t === "object" ? object(value) : t === "array" ? Array.isArray(value) : + t === "integer" ? Number.isSafeInteger(value) : t === "number" ? typeof value === "number" && Number.isFinite(value) : typeof value === t; + if (types.length && !types.some(is)) throw new Error(`${path} must be ${types.join(" or ")}`); + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error(`${path} must be a finite number`); + for (const [key, bad, word] of [["minimum", (n) => value < n, "at least"], ["maximum", (n) => value > n, "at most"], + ["exclusiveMinimum", (n) => value <= n, "greater than"], ["exclusiveMaximum", (n) => value >= n, "less than"]]) { + if (typeof p[key] === "number" && bad(p[key])) throw new Error(`${path} must be ${word} ${p[key]}`); + } + } + if (typeof value === "string") { + const length = [...value].length; + if (p.minLength != null && length < p.minLength) throw new Error(`${path} needs at least ${p.minLength} characters`); + if (p.maxLength != null && length > p.maxLength) throw new Error(`${path} allows at most ${p.maxLength} characters`); + } + if (Array.isArray(value)) { + if (p.minItems != null && value.length < p.minItems) throw new Error(`${path} needs at least ${p.minItems} items`); + if (p.maxItems != null && value.length > p.maxItems) throw new Error(`${path} allows at most ${p.maxItems} items`); + if (p.items) value.forEach((v, i) => validateValue(v, p.items, `${path}[${i}]`, root, depth + 1)); + } else if (object(value)) { + for (const key of p.required || []) if (!own(value, key)) throw new Error(`${path}.${key} is required`); + for (const [key, v] of Object.entries(value)) { + if (own(p.properties, key)) validateValue(v, p.properties[key], `${path}.${key}`, root, depth + 1); + else if (p.additionalProperties === false) throw new Error(`${path}.${key} is not an allowed field`); + else if (object(p.additionalProperties)) validateValue(v, p.additionalProperties, `${path}.${key}`, root, depth + 1); + } + } +} export function schemaForm(schema, { prefix = "sf", values = {}, skip = ["metadata"] } = {}) { - const defs = schema?.$defs || schema?.definitions || {}; - const props = Object.entries(schema?.properties || {}).filter(([k]) => !skip.includes(k)); - const req = new Set(schema?.required || []); - if (!props.length) return `

        No inputs.

        `; - return `
        ${props.map(([k, raw]) => { - const { t, p, int, nullable } = kind({ title: k, ...raw }, defs); - const id = `${prefix}-${k.replace(/[^\w-]/g, "_")}`; - const v = values[k] ?? p.default; - const hint = [req.has(k) ? "required" : nullable ? "optional" : "", p.description || resolve(raw, defs).description || ""].filter(Boolean).join(" · "); - let input; + const root = schema || {}, s = resolveSchema(root, root); + const required = new Set(s.required || []); + const props = Object.entries(s.properties || {}).filter(([k]) => !skip.includes(k) || required.has(k)); + const idBase = esc(prefix); + if (s.anyOf || s.oneOf || s.allOf || !props.length && s.additionalProperties !== false) { + const initial = Object.keys(values).length ? values : s.default; + return ``; + } + if (!props.length) return '

        No inputs.

        '; + return `
        ${props.map(([k, raw], index) => { + const { t, p, nullable } = kind({ title: k, ...raw }, root); + // Schema property names are untrusted and can collide after sanitization. + const id = `${idBase}-${index}`, v = own(values, k) ? values[k] : p.default; + const hint = [required.has(k) ? "required" : "optional; blank uses the server default", p.description || ""].filter(Boolean).join(" · "); if (t === "const") return ""; - if (t === "enum") input = ``; - else if (t === "bool") input = ``; - else if (t === "number") input = ``; - else if (t === "text") input = ``; - else if (t === "string") input = ``; - else input = ``; - return ``; + const attrs = `id="${id}" data-k="${esc(k)}" data-t="${t}" ${required.has(k) ? 'aria-required="true"' : ""} aria-describedby="${id}-hint"`; + let input; + if (t === "enum" || t === "bool") { + const choices = t === "bool" ? [true, false, ...(nullable ? [null] : [])] : p.enum; + input = ``; + } else if (t === "int" || t === "num") input = ``; + else if (t === "str") input = ``; + else input = ``; + return `
        ${esc(hint)}${input}
        `; }).join("")}
        `; } -// the form's values as an object; throws Error(message) on bad JSON or a missing required field export function readForm(el, schema) { - const out = {}; - const req = new Set(schema?.required || []); + const rootInput = el.querySelector("[data-root-json]"); + if (rootInput) { + let value; + try { value = JSON.parse(rootInput.value || "{}"); } catch { throw new Error("Input isn't valid JSON"); } + validateValue(value, schema); + return value; + } + const out = Object.create(null), s = resolveSchema(schema, schema); + const required = new Set(s.required || []); for (const f of el.querySelectorAll("[data-k]")) { - const k = f.dataset.k, t = f.dataset.t; - if (t === "bool") { out[k] = f.checked; continue; } - const raw = f.value; - if (raw === "" || raw == null) { if (req.has(k) && t !== "str") throw new Error(`${k} is required`); if (req.has(k)) out[k] = ""; continue; } - if (t === "int" || t === "num") { const n = Number(raw); if (!Number.isFinite(n)) throw new Error(`${k} should be a number`); out[k] = t === "int" ? Math.trunc(n) : n; } - else if (t === "enum") out[k] = JSON.parse(raw); - else if (t === "json") { try { out[k] = JSON.parse(raw); } catch { throw new Error(`${k} isn't valid JSON`); } } - else out[k] = raw; + const k = f.dataset.k, t = f.dataset.t, raw = f.value; + if (raw === "" || raw == null) { + if (required.has(k)) { + if (t !== "str" && t !== "text") throw new Error(`${k} is required`); + out[k] = ""; + } + continue; + } + if (t === "int" || t === "num") { + const n = Number(raw); + if (!Number.isFinite(n)) throw new Error(`${k} must be a finite number`); + if (t === "int" && !Number.isSafeInteger(n)) throw new Error(`${k} must be a whole number within the safe integer range`); + out[k] = n; + } else if (["enum", "bool", "json"].includes(t)) { + try { out[k] = JSON.parse(raw); } catch { throw new Error(`${k} isn't valid JSON`); } + } else out[k] = raw; + } + for (const [k, raw] of Object.entries(s.properties || {})) { + const p = resolveSchema(raw, schema); + if (own(p, "const") && !own(out, k)) out[k] = p.const; } - // a const field (an action's type discriminator) goes in as its value - for (const [k, p] of Object.entries(schema?.properties || {})) if (p?.const !== undefined && !(k in out)) out[k] = p.const; + validateValue(out, schema); return out; } -export function wireForm(el) { - el.addEventListener("change", (e) => { - const c = e.target.closest('[data-t="bool"]'); - if (c) c.nextElementSibling.textContent = String(c.checked); - }); -} +export function wireForm() { /* Native labelled controls handle their own state. */ } diff --git a/web/js/space-media.js b/web/js/space-media.js new file mode 100644 index 0000000000000000000000000000000000000000..3d9590001f9bce71d88117e573b523306d69ec53 --- /dev/null +++ b/web/js/space-media.js @@ -0,0 +1,15 @@ +// Asset paths are relative to the Space, never the explorer. Resolve only +// same-origin HTTPS paths; arbitrary URL schemes and foreign hosts are rejected. +export function spaceMedia(data, host) { + if (!data || typeof data !== "object") return data; + if (Array.isArray(data)) return data.map((v) => spaceMedia(v, host)); + const out = Object.fromEntries(Object.entries(data).map(([k, v]) => [k, spaceMedia(v, host)])); + if (typeof out.asset_path === "string" && out.asset_path.startsWith("/assets/") && /^(audio|image|video)\//.test(out.mime || "")) { + try { + const base = new URL(host), asset = new URL(out.asset_path, base); + if (base.protocol === "https:" && asset.origin === base.origin && !asset.username && !asset.password) + out.media_url = asset.href; + } catch { /* malformed source: keep its metadata visible */ } + } + return out; +} diff --git a/web/js/space-task-view.js b/web/js/space-task-view.js new file mode 100644 index 0000000000000000000000000000000000000000..28777bf79f8bea3f242beab38662f949cf2d6222 --- /dev/null +++ b/web/js/space-task-view.js @@ -0,0 +1,15 @@ +import { esc, fmt } from "./util.js"; +import { value } from "./render.js"; + +// Keep the task's actual inputs visible; hashes and bookkeeping belong in an +// expandable record. All content still passes through the shared safe renderer. +export function taskContent(task) { + const primary = /^(prompt|instruction|instructions|question|description|messages|input|inputs|content|text|audio|audio_url|image|image_url|video_url|context|tools)$/i; + const entries = Object.entries(task); + const inputs = entries.filter(([key, v]) => primary.test(key) && v != null && v !== ""); + const metadata = Object.fromEntries(entries.filter(([key]) => !primary.test(key))); + return `${task.media_ready === false ? `
        Media loads when the episode starts. Start this task to ask the Space to prepare its asset. The player appears with the observation when the server provides it.
        ` : ""} +
        ${inputs.map(([key, v]) => `

        ${esc(key.replaceAll("_", " "))}

        ${value(v, key, 0, task)}
        `).join("") || `

        This task exposes structured fields. Open its record below to inspect them.

        `}
        +
        Task record · ${fmt.format(Object.keys(metadata).length)} fields + ${value(metadata)}
        `; +} diff --git a/web/js/space.js b/web/js/space.js index 1997d19f2de70138380301cdaf02d281812e04cd..fc9eff54338064ff5f6dc4b73b6651cca4735a07 100644 --- a/web/js/space.js +++ b/web/js/space.js @@ -6,13 +6,16 @@ // The page: an Interface table (how an agent acts, what it sees, the reward, tasks, web app, episodes), then the // playground (awake only), its app, tools, tasks, rewards, schemas, files, connecting, `openenv validate`'s criteria, // and the README. Status, wake and restart sit beside it. -import { $, api, esc, md, ago, sk, emptyState, toast, fmt, spinner, storage, setMeta } from "./util.js"; +import { $, api, esc, md, ago, sk, emptyState, toast, fmt, spinner, storage, setMeta, ownerLink } from "./util.js"; import { icon } from "./icons.js"; import { AGENTS, agentHtml, agentName, chosenAgent, copyBtn, snippet } from "./connect.js"; import { getSession } from "./session.js"; import { schemaForm, readForm, wireForm } from "./schemaform.js"; import { value } from "./render.js"; import { fileViewer } from "./files.js"; +import { supportPanel } from "./support.js"; +import { taskContent } from "./space-task-view.js"; +import { spaceMedia } from "./space-media.js"; const enc = (s) => s.split("/").map(encodeURIComponent).join("/"); const STAGE = { @@ -22,7 +25,7 @@ const STAGE = { }; const stageOf = (x) => STAGE[x] || [String(x || "unknown").toLowerCase().replace(/_/g, " "), "faint"]; const STARTING = new Set(["APP_STARTING", "BUILDING", "RUNNING_BUILDING", "RUNNING_APP_STARTING"]); -const KIND = { harbor: "OpenEnv × Harbor", openenv: "OpenEnv", "nemo-gym": "NeMo Gym server", ors: "ORS server", mcp: "MCP server", api: "HTTP API" }; +const KIND = { harbor: "OpenEnv × Harbor", openenv: "OpenEnv", "nemo-gym": "NeMo Gym server", gymnasium: "Gymnasium HTTP server", ors: "ORS server", mcp: "MCP server", api: "HTTP API" }; const shortSplit = (n) => (String(n).includes("/") ? String(n).split("/").filter(Boolean).pop() : String(n)); const IMAGE = /((^|_)(image|img|frame|screenshot|pixels|png|jpe?g|picture|photo|render)s?(_?(b64|data|url|bytes))?$|base64|(^|_)b64$)/i; const HIDDEN_OBS = new Set(["reward", "done", "metadata"]); @@ -179,7 +182,7 @@ function interfaceTable(me) { if (!me.L && !me.F) return ifaceSkeleton(); const rows = interfaceRows(me); const pending = !me.F || !me.L; - return `
        ${rows.map(([k, vals]) => `
        ${esc(k)}
        ${vals.length ? vals.join("") : pending ? sk.line(40) : NONE(emptyFor(me, k))}
        `).join("")}
        + return `${supportPanel(me.L?.support)}
        ${rows.map(([k, vals]) => `
        ${esc(k)}
        ${vals.length ? vals.join("") : pending ? sk.line(40) : NONE(emptyFor(me, k))}
        `).join("")}
        ${me.F?.error ? `

        ${icon("alert", 12)} Couldn't read its files: ${esc(me.F.error)}

        ` : ""}`; } const emptyFor = (me, k) => (k === "Tasks" ? "none declared" : k === "Web app" ? "none declared" : !me.L?.running && !seenOf(me) ? "not declared; wake it to ask its server" : "none"); @@ -229,7 +232,7 @@ function appSection(me) { } // what the playground can do here: call its MCP tools, take actions (reset/step), or call its own HTTP routes -const modesOf = (L) => [...(L.mcp ? ["tools"] : []), ...(L.step_api && !L.mcp ? ["action"] : []), ...(L.api?.length ? ["routes"] : [])]; +const modesOf = (L) => [...(L.mcp?.length ? ["tools"] : []), ...(L.step_api && !L.mcp?.length ? ["action"] : []), ...(L.api?.length ? ["routes"] : [])]; const MODE = { tools: "Its tools", action: "Actions", routes: "Its routes" }; function playSection(me) { @@ -261,7 +264,7 @@ function playground(L) { action: () => `
        Action
        ${schemaForm(L.schema?.action || {}, { prefix: "pga" })}
        `, routes: () => ` -

        +

        `, }; return `
        @@ -270,7 +273,7 @@ function playground(L) { ${L.openenv && L.step_api ? `${resetOpts}` : ""} ${modes.length > 1 ? `${modes.map((m, i) => ``).join("")}` : ""} ${modes.map((m, i) => `
        ${pane[m]()}
        `).join("")} -

        On the Space itself, in a session of its own (cookies included, kept on the explorer) that ends after 10 idle minutes. Its owner pays for its hardware, not you.

        +

        Requests go to the Space itself. ${L.openenv ? "An OpenEnv WebSocket keeps your episode together." : L.api?.length ? "HTTP cookies are isolated per explorer session; the server controls episode state." : "Tool state depends on the server's MCP implementation."} Sessions end after 10 idle minutes. The Space's owner pays for its hardware.

        `; @@ -278,7 +281,7 @@ function playground(L) { function tasksSection(me) { const P = probeOf(me), d = D(me); - if (me.L?.running && me.L.task_api?.splits?.length) return taskBrowser(me.L); + if (me.L?.running && me.L.task_api) return taskBrowser(me); const out = []; if (P?.task_api && !P.task_api.splits?.length) out.push(`
        Its Task API lists no splits${probeSrc(me)}
        `); const splits = P?.task_api?.splits || []; @@ -304,11 +307,20 @@ function tasksSection(me) { return out.join("") || `

        ${me.L?.running ? "No Task API, and no tasks declared in its files." : "No tasks declared in its files."}${!me.L?.running && !seenOf(me) ? " Its server may serve some: wake it to ask." : ""}

        `; } -function taskBrowser(L) { - const splits = L.task_api.splits; - const first = Math.max(0, splits.findIndex((y) => y.default)); +function taskCatalog(me) { + const api = me.L.task_api, environments = api.environments || [api]; + const selected = environments.find((e) => e.env === me.taskPage.env) || environments[0]; + me.taskPage.env = selected.env; + return selected; +} + +function taskBrowser(me) { + const { splits } = taskCatalog(me), envs = me.L.task_api.environments || [me.L.task_api]; + if (!splits.some((s) => s.name === me.taskPage.split)) me.taskPage.split = (splits.find((s) => s.default) || splits[0])?.name || ""; return `
        -
        ${splits.map((x, i) => ``).join("")} + ${envs.length > 1 ? `` : ""} +

        Choose a split, then a task to inspect its inputs. Counts come from this Space's Task API.

        +
        ${splits.map((x) => ``).join("")}
        ${sk.lines(90, 90, 90, 90)}
        @@ -317,7 +329,12 @@ function taskBrowser(L) { function toolsSection(me) { const P = probeOf(me), d = D(me); - if (P?.mcp?.length) return `${me.L?.running ? "" : `
        ${fmt.format(P.mcp.length)} tools${probeSrc(me)}
        `}${toolsRef(P.mcp, !!me.L?.running)}`; + if (P?.mcp?.length) { + const own = new Set(P.mcp.map((t) => t.name)); + const extra = (P.bridge_tools || []).filter((t) => !own.has(t.name)); + return `${me.L?.running ? "" : `
        ${fmt.format(P.mcp.length)} tools${probeSrc(me)}
        `}${toolsRef(P.mcp, !!me.L?.running)} + ${extra.length ? `
        Also available through the explorer's bridge
        ${toolsRef(extra, false)}` : ""}`; + } if (!me.F && !P) return sk.lines(70, 50, 60); const e = d.environment; if (e?.tools?.length) { @@ -329,9 +346,8 @@ function toolsSection(me) { if (tools?.length) return `
        Declared${SRC.file(me, where)}

        ${codes(tools, 40)}

        `; if (P?.bridge_tools?.length) { // a reset/step server: its tools are its episode calls, with its own action's fields const [org, name] = me.spec.split("/"); - return `
        ${fmt.format(P.bridge_tools.length)} tools, through reset and step${SRC.live()}
        -

        Its server has no MCP tools of its own: an agent acts with reset and step, and - step takes its action's fields. Over MCP, this explorer's bridge offers exactly these + return `

        ${fmt.format(P.bridge_tools.length)} tools, through ${P.openenv ? "reset and step" : "its HTTP API"}${SRC.live()}
        +

        ${P.openenv ? "The bridge exposes reset, step and state with the environment's action schema." : "The bridge exposes the server's published routes. Path parameters, query parameters and body have separate inputs."} These are the available tools (/mcp/${esc(org)}/${esc(name)}, under Connect); the playground calls the same ones.

        ${toolsRef(P.bridge_tools, false)}`; } @@ -479,6 +495,10 @@ function schemasSection(me) { if (schema && (schema.action || schema.observation)) { return `
        From its /schema${probeSrc(me)}
        ${schemaTable("Action", schema.action) + schemaTable("Observation", schema.observation) + schemaTable("State", schema.state)}`; } + if (P?.api?.length) return `
        From its OpenAPI${probeSrc(me)}
        ` + P.api.map((r) => + `
        ${esc(r.method)} ${esc(r.path)}${r.summary ? `

        ${esc(r.summary)}

        ` : ""} + ${r.query_schema?.properties && Object.keys(r.query_schema.properties).length ? schemaTable("Query parameters", r.query_schema) : ""} + ${r.schema ? (schemaTable("Request body", r.schema) || snippet(JSON.stringify(r.schema, null, 2))) : '

        No request body schema published.

        '}
        `).join(""); if (!me.F) return sk.lines(70, 50, 60); const m = d.models; if (!m) return `

        No models.py in its files.${!me.L?.running ? " Wake it to ask its server for /schema." : ""}

        `; @@ -518,16 +538,18 @@ function connectPane(me, tab) { if (tab === "agents") { const url = `${location.origin}/mcp/${spec}`; const own = (P?.mcp || []).map((t) => t.name); - const routeTools = L?.running && !own.length ? (L.api || []).map((r) => r.path.replace(/[^A-Za-z0-9]+/g, "_").replace(/^_+|_+$/g, "").toLowerCase()).map((b, i) => (L.api[i].method === "POST" || b.startsWith("get_") ? b : `get_${b}`)) : []; + const routeTools = L?.running ? (L.api || []).map((r) => r.path.replace(/[^A-Za-z0-9]+/g, "_").replace(/^_+|_+$/g, "").toLowerCase()).map((b, i) => (L.api[i].method === "POST" || b.startsWith("get_") ? b : `get_${b}`)).filter((n) => !own.includes(n)) : []; const openenv = P?.openenv || P?.step_api; - const tools = P ? [...own, ...(openenv && P.step_api && !own.includes("reset") ? ["reset", ...(own.length ? [] : ["step"]), "state"] : []), ...routeTools, ...(P.task_api ? ["list_splits", "list_tasks", "get_task"] : [])] : []; + const tools = P?.bridge_tools ? P.bridge_tools.map((t) => t.name) : P ? [...own, ...(openenv && P.step_api && !own.includes("reset") ? ["reset", ...(own.length ? [] : ["step"]), "state"] : []), ...routeTools, ...(P.task_api ? ["list_splits", "list_tasks", "get_task"] : [])] : []; const chosen = chosenAgent(); return `
        MCP${esc(url)}${copyBtn(url)}
        -

        OpenEnv servers answer tools/list and tools/call but not MCP's handshake, so agents connect through this bridge. Each agent session gets an episode of its own on the Space${tools.length ? `, with these tools: ${tools.slice(0, 16).map((t) => `${esc(t)}`).join(" ")}${tools.length > 16 ? " …" : ""}` : ""}${L && !L.running ? "; a sleeping Space is woken on the first call" : ""}. Images in observations come back as images.

        +

        Connect through the explorer's MCP bridge. ${openenv ? "An OpenEnv WebSocket keeps each agent's episode together." : "Each agent connection has its own HTTP cookies and MCP session; the server controls episode state."}${tools.length ? ` Available tools: ${tools.slice(0, 16).map((t) => `${esc(t)}`).join(" ")}${tools.length > 16 ? " …" : ""}.` : ""}${L && !L.running ? " A sleeping Space is woken on the first call." : ""} Images in observations come back as images.

        ${AGENTS.filter(([id]) => id !== "python").map(([id, label]) => ``).join("")}
        ${agentHtml(chosen === "python" ? "claude" : chosen, agentName(spec), url)}
        `; } if (tab === "python") { + if (P && !P.openenv) return `

        Use the same MCP bridge from Python:

        ${snippet(agentHtmlText(spec))} + ${P.api?.length ? `

        Or inspect the HTTP schema directly. Reuse the client for subsequent calls so the server's cookies persist:

        ${snippet(`# pip install httpx\nimport httpx\n\nwith httpx.Client(base_url=${JSON.stringify(host)}, timeout=120) as client:\n schema = client.get("/openapi.json").json()\n print(schema["paths"])\n # Choose a route and supply its required path, query and body fields here.`)}` : ""}`; const action = actionExample(me); const generic = `# pip install openenv-core (no package of its own: plain dictionaries) from openenv.core.generic_client import GenericEnvClient @@ -558,6 +580,7 @@ env = AutoEnv.from_env("${spec}")` : null; ${snippet(`docker run -it -p 8000:${port} --platform=linux/amd64 registry.hf.space/${sub}:latest`)}

        Then http://localhost:8000${esc(path)}, or point a client at http://localhost:8000. Secrets it needs (its own API keys) go in with -e NAME=value.

        `; } + if (!L?.openenv_verified) return `

        OpenEnv compatibility has not been verified. To copy this Space, use its Hub repository:

        ${icon("external", 14)}Duplicate on Hugging Face`; return `

        A copy of this Space on your account, to change and run as your own (needs hf auth login):

        ${snippet(`pip install openenv-core\nopenenv fork ${spec}`)}

        Options: --repo-id you/name, --private, --hardware cpu-upgrade, --set-env KEY=VALUE, --set-secret KEY=VALUE.

        `; @@ -583,11 +606,16 @@ function actionExample(me) { function conformanceSection(me) { const L = me.L; if (!L) return sk.lines(70, 70, 70, 70, 70, 70); + const P = probeOf(me); + const declared = me.s.declared_openenv || me.s.openenv || me.s.framework === "openenv" || me.s.manifest || D(me).manifest; + if (P && !P.openenv && !declared) return `

        This server exposes ${esc(P.support?.framework?.label || KIND[P.framework] || "an HTTP API")}. Its published interface is shown above; OpenEnv-specific checks do not apply.

        `; const rows = L.running ? L.conformance : seenOf(me)?.conformance; const where = L.running ? `From the probe ${L.checked ? esc(ago(L.checked)) : "just now"}.` : seenOf(me) ? `From when it was last seen running, ${esc(ago(seenOf(me).checked_at))}.` : ""; if (!rows?.length) return `

        ${L.running ? "Not checked." : "Not seen running here yet: wake it, and its server is checked against these."}

        ${criteriaList()}`; const host = L.host; - return `

        ${where} The runtime criteria of openenv validate, judged from the explorer's own requests.

        + return `${P && !P.openenv ? `

        This repository has OpenEnv-related metadata, but its server did not return the expected OpenEnv schemas in this probe. The checks below show what answered.

        ` : ""} +

        ${where} Basic endpoint checks corresponding to openenv validate. Passing does not verify an episode, reward correctness, or training quality.

        +

        The MCP endpoint check accepts JSON-RPC error responses. ${P?.mcp?.length ? `Tool discovery returned ${fmt.format(P.mcp.length)} tools; tool execution is untested by this probe.` : `No usable MCP tools were discovered.${P?.mcp_error ? ` Server reply: ${esc(P.mcp_error)}.` : ""}`} OpenEnv environments can use typed actions without MCP tools.

        ${rows.map((c) => ``).join("")}
        CriterionResultWhy
        ${esc(c.id)}${esc(c.label)}${esc(c.status)}${esc(c.reason)}
        ${host ? `

        Run it yourself:

        ${snippet(`openenv validate --url ${host}`)}` : ""}`; @@ -645,7 +673,7 @@ function describe(me) { function factsLine(me) { const { s, L } = me, d = D(me), P = probeOf(me); const [label, tone] = stageOf(L?.stage || s.stage); - const kind = KIND[P?.framework] || (s.openenv || d.manifest || d.app?.factory ? "OpenEnv" : "Docker Space"); + const kind = L?.openenv_verified ? (L.harbor ? "OpenEnv × Harbor" : "OpenEnv · API checked") : P?.framework && !["openenv", "harbor"].includes(P.framework) ? KIND[P.framework] : s.declared_openenv || s.openenv || d.manifest ? "Unverified Space" : "Docker Space"; const mode = P?.mode || (d.dockerfile?.mode ? String(d.dockerfile.mode).toLowerCase() : d.app?.mode ? String(d.app.mode).toLowerCase() : null); const py = d.pyproject && !d.pyproject.error && d.pyproject.openenv ? `${d.pyproject.openenv_package || "openenv-core"} ${d.pyproject.openenv_version || ""}`.trim() : s.openenv_version ? `openenv ${s.openenv_version}` : null; const hw = s.hardware || me.F?.hardware; @@ -670,7 +698,7 @@ function linksLine(me) { const SECTIONS = [ ["interface", "Interface", "gauge"], ["play", "Playground", "play"], ["harbor", "Rollouts here", "box"], ["app", "App", "window"], ["tools", "Tools", "wrench"], ["tasks", "Tasks", "list"], ["rewards", "Rewards", "target"], ["schema", "Schemas", "code"], ["files", "Files", "folder"], - ["connect", "Connect", "plug"], ["conformance", "Conformance", "check"], ["readme", "README", "doc"], + ["connect", "Connect", "plug"], ["conformance", "API checks", "check"], ["readme", "README", "doc"], ]; function sectionBody(me, id) { @@ -705,7 +733,7 @@ const sectionsOf = (me) => SECTIONS.filter(([id]) => id !== "harbor" || (me.L?.r function skeleton(spec) { const [org, name] = spec.split("/"); return `
        -

        ${esc(org)}/${esc(name)}

        ${sk.line(55, 14)}
        ${sk.line(40, 13)}
        +

        ${ownerLink(org, "/")}${esc(name)}

        ${sk.line(55, 14)}
        ${sk.line(40, 13)}

        ${icon("gauge", 15)}Interface

        ${ifaceSkeleton()}

        ${icon("play", 15)}Playground

        ${sk.box(160)}
        @@ -717,15 +745,27 @@ export async function mount(el, { spec }) { el.innerHTML = skeleton(spec); const me = page = { spec, s: null, F: null, L: null, el, session: null, log: [], episode: null, poll: null, started: null, taskPage: { split: null, start: 0 }, rank: null, rewards: [], appOpen: false, connectTab: storage.get("sp-ctab") || "agents", filePath: null, viewer: null, rendered: false }; + const params = new URLSearchParams(location.search), index = Number(params.get("task")); + if (params.has("task") && Number.isSafeInteger(index) && index >= 0) { + me.taskPage = { env: params.get("env"), split: params.get("split"), start: Math.floor(index / 20) * 20 }; + me.linkedTask = index; + } me.onKey = (e) => { if (e.key === "Escape") $(".sp-app.full", el)?.classList.remove("full"); }; document.addEventListener("keydown", me.onKey); el.addEventListener("click", (e) => onClick(me, e)); - el.addEventListener("change", (e) => { if (e.target.id === "pg-tool") showTool(me); }); + el.addEventListener("change", (e) => { + if (e.target.id === "pg-tool") showTool(me); + if (e.target.id === "tk-env") { + me.taskPage = { env: e.target.value, split: null, start: 0 }; + $("#sb-tasks", me.el).innerHTML = taskBrowser(me); + loadTasks(me); + } + }); // all three at once: the card (and README), what its files declare, what its server says (or said) const card = api(`/api/spaces/${enc(spec)}`); loadFiles(me); loadLive(me, false); - api(`/api/search/rank?key=${encodeURIComponent(`space:${spec}`)}`).then((rk) => { // where it stands among environment Spaces, by trending + api(`/api/search/rank?scope=ready&key=${encodeURIComponent(`space:${spec}`)}`).then((rk) => { // rank among recently checked environment Spaces if (page === me) { me.rank = rk; const g = $("#sp-glance", el); if (g) g.innerHTML = glance(me); } }).catch(() => {}); // not in the catalog yet: no rank shown try { me.s = await card; } catch (e) { @@ -742,17 +782,18 @@ function render(me) { if (!s) return; const [org, name] = spec.split("/"); const sections = sectionsOf(me); - const kind = s.framework === "openenv" || s.openenv ? "OpenEnv Space" : s.framework === "ors" ? "ORS Space" : "environment Space"; // as the server-rendered title says + const kind = me.L?.openenv_verified ? "OpenEnv Space" : s.declared_openenv || s.openenv || s.manifest ? "Unverified Space" : s.framework === "ors" ? "ORS Space" : "environment Space"; // as the server-rendered title says setMeta({ title: `${s.heading || name} · ${kind}`, description: `${spec}: an RL environment Space on Hugging Face. ${describe(me)} See it live: its app, a playground with rewards, its tasks, and MCP for coding agents.` }); me.viewer?.destroy(); me.viewer = null; el.innerHTML = `
        -

        ${esc(org)}/${esc(name)}

        +

        ${ownerLink(org, "/")}${esc(name)}

        ${esc(describe(me))}

        ${factsLine(me)}
        -
        + + ${(s.declared_openenv || s.openenv || s.manifest) && !me.L?.openenv_verified ? `
        OpenEnv API not verified. This Space was discovered from repository metadata. A file named openenv.yaml or a Hub tag does not establish compatibility.${me.L?.running ? " The current server did not pass the required API checks." : " It needs a successful check while running."}
        ` : ""}
        ${sections.map(([id, title, ic]) => `

        ${icon(ic, 15)}${esc(title)}

        @@ -815,7 +856,7 @@ async function loadLive(me, fresh) { // the page is drawn again only when what the server offers changed (the first answer, or it woke up); otherwise // just the parts that say so, so a playground in progress keeps its episode if (!me.rendered) { /* render() runs when the card arrives and reads me.L */ } - else if (!was || was.error || was.running !== L.running) { const y = scrollY; render(me); scrollTo(0, y); } + else if (!was || was.error || was.running !== L.running || was.openenv_verified !== L.openenv_verified) { const y = scrollY; render(me); scrollTo(0, y); } else if (!L.running && was.stage !== L.stage) paint(me, ["play", "app", "interface", "conformance"]); else paint(me, ["conformance"]); if (STARTING.has(L.stage)) { me.started ||= Date.now(); schedule(me, 4000); } @@ -934,7 +975,7 @@ async function onClick(me, e) { } const sp = t.closest("[data-split]"); if (sp) { - me.taskPage = { split: sp.dataset.split, start: 0 }; + me.taskPage = { env: me.taskPage.env, split: sp.dataset.split, start: 0 }; me.el.querySelectorAll("[data-split]").forEach((x) => x.setAttribute("aria-pressed", String(x === sp))); $("#tk-detail", me.el).hidden = true; loadTasks(me); @@ -953,6 +994,8 @@ async function onClick(me, e) { await reset(me); return; } + const nextTask = t.closest("[data-task-move]"); + if (nextTask) { openTask(me, me.selectedTask + Number(nextTask.dataset.taskMove)); return; } if (t.closest("#tk-close")) { $("#tk-detail", me.el).hidden = true; me.el.querySelectorAll("[data-task]").forEach((r) => r.classList.remove("on")); return; } const row = t.closest("[data-task]"); if (row) openTask(me, +row.dataset.task); @@ -1000,6 +1043,8 @@ function showRoute(me) { $("#pg-rdesc", me.el).textContent = r.summary || ""; $("#pg-rparams", me.el).innerHTML = r.params.length ? `
        ${r.params.map((p) => ``).join("")}
        ` : ""; + $("#pg-query", me.el).innerHTML = Object.keys(r.query_schema?.properties || {}).length ? + `
        Query parameters
        ${schemaForm(r.query_schema, { prefix: "pgq", skip: [] })}` : ""; $("#pg-rform", me.el).innerHTML = r.method === "POST" ? (r.schema ? schemaForm(r.schema, { prefix: "pgr", skip: [] }) : ``) : ""; } @@ -1008,25 +1053,36 @@ async function sendRoute(me) { const r = me.L.api[+$("#pg-route", me.el).value]; const params = {}; for (const i of me.el.querySelectorAll("[data-rp]")) { - if (!/^[\w.\-~]{1,200}$/.test(i.value)) { toast(`${i.dataset.rp}: letters, digits, . - _ only`); return; } + if (!/^[\w.\-~]{1,200}$/.test(i.value) || [".", ".."].includes(i.value)) { toast(`${i.dataset.rp}: enter a path value using letters, digits, . - _`); return; } params[i.dataset.rp] = i.value; } - let body = null; + let body = null, query = {}; + try { if (r.query_schema) query = readForm($("#pg-query", me.el), r.query_schema); } + catch (e) { toast(e.message); return; } if (r.method === "POST") { try { if (r.schema) body = readForm($("#pg-rform", me.el), r.schema); else body = JSON.parse($("#pg-rbody", me.el)?.value || "{}"); } catch (e) { toast(e.message || "The body isn't valid JSON"); return; } } - await send(me, "http", { method: r.method, path: r.path, params, body }, `${r.method} ${r.path}`); + await send(me, "http", { method: r.method, path: r.path, params, query, body }, `${r.method} ${r.path}`); } async function ensureSession(me) { - if (!me.session) me.session = (await api(`/api/spaces/${enc(me.spec)}/play`, { method: "POST", body: { op: "start" } })).session; + if (!me.session) { + const generation = me.sessionGeneration || 0; + const id = (await api(`/api/spaces/${enc(me.spec)}/play`, { method: "POST", body: { op: "start" } })).session; + if (page !== me || generation !== (me.sessionGeneration || 0)) { + api(`/api/spaces/${enc(me.spec)}/play`, { method: "POST", body: { op: "end", session: id } }).catch(() => {}); + throw new Error("This playground was closed."); + } + me.session = id; + } return me.session; } function endSession(me) { + me.sessionGeneration = (me.sessionGeneration || 0) + 1; const id = me.session; me.session = null; me.episode = null; me.log = []; if (id) api(`/api/spaces/${enc(me.spec)}/play`, { method: "POST", body: { op: "end", session: id } }).catch(() => {}); @@ -1034,6 +1090,8 @@ function endSession(me) { } async function send(me, op, data, label) { + if (me.sending) return; + me.sending = true; const entry = { op, label, pending: true }; if (op === "reset") me.log = []; me.log.push(entry); @@ -1042,15 +1100,17 @@ async function send(me, op, data, label) { const post = () => api(`/api/spaces/${enc(me.spec)}/play`, { method: "POST", body: { op, session: me.session, data } }); try { await ensureSession(me); + if (!me.log.includes(entry)) return; let r; try { r = await post(); } catch (err) { - if (err.status !== 410) throw err; + if (err.status !== 410 || !me.log.includes(entry)) throw err; me.session = null; // it timed out: a fresh session (an episode in progress is gone, so a step has to start over) - if (op === "step") { me.episode = null; throw new Error("The session ended after 10 idle minutes. Start a new episode."); } + if (op !== "reset") { me.episode = null; throw new Error("The session ended. Start a new episode before continuing."); } await ensureSession(me); r = await post(); } - Object.assign(entry, { pending: false, result: r.result, ms: r.ms, stateful: r.stateful }); + if (page !== me || !me.log.includes(entry)) return; + Object.assign(entry, { pending: false, result: spaceMedia(r.result, me.L.host), ms: r.ms, stateful: r.stateful }); if (op === "reset") me.episode = { steps: 0, total: 0, done: false, task: taskOf(r.result) }; else { me.episode ||= { steps: 0, total: 0, done: false, task: null }; @@ -1062,15 +1122,16 @@ async function send(me, op, data, label) { } catch (err) { Object.assign(entry, { pending: false, error: err.message }); } finally { - busy(me, false); - renderLog(me); + me.sending = false; + if (page === me) { busy(me, false); renderLog(me); } } } const rewardOf = (op, r) => (op === "http" ? (typeof r?.json?.reward === "number" ? r.json.reward : null) : op === "call" ? (typeof r?.structuredContent?.reward === "number" ? r.structuredContent.reward : typeof r?.structured_content?.reward === "number" ? r.structured_content.reward : null) : r?.reward ?? r?.observation?.reward); -const doneOf = (r) => r?.done ?? r?.observation?.done; +const doneOf = (r) => Boolean(r?.done || r?.observation?.done || r?.terminated || r?.truncated || + r?.json?.done || r?.json?.terminated || r?.json?.truncated || r?.structuredContent?.done); const taskOf = (r) => { const o = r?.observation || {}; return o.task_id || o.metadata?.task_id || (o.index != null ? `#${o.index}` : null); }; -const busy = (me, on) => me.el.querySelectorAll("#pg-reset, #pg-step, #pg-call, #pg-send").forEach((b) => (b.disabled = on)); +const busy = (me, on) => me.el.querySelectorAll("#pg-reset, #pg-step, #pg-call, #pg-send, [data-play-task]").forEach((b) => (b.disabled = on)); function resetParams(me) { const out = {}; @@ -1084,7 +1145,7 @@ function resetParams(me) { const split = $("#pg-split", me.el)?.value, idx = $("#pg-index", me.el)?.value; if (split) out.split = split; if (idx !== "" && idx != null) { - if (!(Number(idx) >= 0)) throw new Error("The task index should be 0 or more"); + if (!Number.isSafeInteger(Number(idx)) || Number(idx) < 0) throw new Error("The task index should be a whole number, 0 or more"); out[me.indexKey || "index"] = Number(idx); // servers name it their own way: the key their Task API rows use } return out; @@ -1127,7 +1188,7 @@ function renderLog(me) { : `No episode yet`; } if (!me.log.length) { - box.innerHTML = `
        ${icon("play", 22)}${me.L?.openenv ? "Start an episode" : "Call a tool"}

        Each ${me.L?.openenv ? "reset, step and tool call" : "tool call"} shows here with what the environment sent back and the reward it gave.

        `; + box.innerHTML = `
        ${icon("play", 22)}${me.L?.openenv ? "Start an episode" : me.L?.api?.length ? "Send a request" : "Call a tool"}

        Each request shows here with the server's response and any reward it returns.

        `; return; } box.innerHTML = me.log.map((x, i) => { @@ -1139,7 +1200,7 @@ function renderLog(me) { : x.error ? `
        ${icon("alert")}${esc(x.error)}
        ` : x.op === "call" ? callView(x.result) : x.op === "http" ? httpView(x.result) : obsView(x.result); return `
        ${head}
        ${body}
        `; - }).join("") + (me.log.at(-1)?.stateful === false ? `

        This server allows one session at a time and it's taken, so each tool call here stands alone.

        ` : ""); + }).join("") + (me.L?.openenv && me.log.at(-1)?.stateful === false ? `

        The WebSocket session is unavailable. These tool calls use HTTP and may not retain episode state.

        ` : ""); } // ── what the environment sent back ─────────────────────────────────────────── @@ -1149,7 +1210,7 @@ function renderLog(me) { const STATUS_KEYS = ["feedback", "message", "status", "result", "error", "info", "action_result"]; const ORDER = ["prompt", "instruction", "question", "task", "observation", "text", "content", "dashboard", "output", "stdout", "stderr"]; const isObj = (v) => v && typeof v === "object" && !Array.isArray(v); -const isMedia = (html) => /^<(img|audio)\b/.test(html); +const isMedia = (html) => /^<(img|audio|video)\b/.test(html); let foldN = 0; function textBlock(v) { @@ -1254,26 +1315,29 @@ function httpView(r) { async function loadTasks(me) { const L = me.L, box = $("#tk-table", me.el); if (!box || !L?.task_api) return; - const splits = L.task_api.splits; + const splits = taskCatalog(me).splits; me.taskPage.split ??= (splits.find((x) => x.default) || splits[0])?.name || ""; - const { split, start } = me.taskPage; + const { split, start, env } = me.taskPage; + me.selectedTask = null; + $("#tk-detail", me.el).hidden = true; const total = splits.find((x) => x.name === split)?.num_tasks; box.innerHTML = sk.lines(90, 90, 90, 90); $("#tk-range", me.el).textContent = ""; try { - const r = await api(`/api/spaces/${enc(me.spec)}/tasks?split=${encodeURIComponent(split)}&start=${start}&stop=${start + 20}`); - if (page !== me || me.taskPage.split !== split || me.taskPage.start !== start) return; + const r = await api(`/api/spaces/${enc(me.spec)}/tasks?env=${encodeURIComponent(env)}&split=${encodeURIComponent(split)}&start=${start}&stop=${start + 20}`); + if (page !== me || me.taskPage.env !== env || me.taskPage.split !== split || me.taskPage.start !== start) return; me.tasks = r.tasks; if (r.tasks.length && me.indexKey == null) { me.indexKey = "task_index" in r.tasks[0] ? "task_index" : "index"; L.indexKey = me.indexKey; } const keys = columns(r.tasks); box.innerHTML = r.tasks.length ? `
        ${keys.map((k) => ``).join("")}${r.tasks.map((t, i) => - `${keys.map((k) => ``).join("")}`).join("")}
        ${esc(k)}
        ${cellOf(t[k])}
        + `${keys.map((k) => `${cellOf(t[k])}`).join("")}`).join("")}
        ${r.withheld.length ? `

        ${icon("shield", 12)} Left out, as they may hold the answer: ${r.withheld.map((k) => `${esc(k)}`).join(", ")}.

        ` : ""}` : `

        No tasks here.

        `; $("#tk-range", me.el).textContent = r.tasks.length ? `${fmt.format(start + 1)}–${fmt.format(start + r.tasks.length)}${total != null ? ` of ${fmt.format(total)}` : ""}` : ""; $("#tk-prev", me.el).disabled = start === 0; $("#tk-next", me.el).disabled = r.tasks.length < 20 || (total != null && start + 20 >= total); box.querySelectorAll("[data-task]").forEach((tr) => tr.addEventListener("keydown", (e) => { if (e.key === "Enter") openTask(me, +tr.dataset.task); })); + if (me.linkedTask != null) { const target = me.linkedTask; me.linkedTask = null; openTask(me, target - start); } } catch (e) { box.innerHTML = `
        ${icon("alert")}${esc(e.message)}
        `; } } @@ -1281,26 +1345,45 @@ function columns(rows) { const first = ["task_index", "index", "task_id", "id", "task_name", "name", "title"]; const keys = [...new Set(rows.flatMap((r) => Object.keys(r)))]; const short = keys.filter((k) => rows.every((r) => r[k] == null || (["string", "number", "boolean"].includes(typeof r[k]) && String(r[k]).length <= 80))); - const pick = [...first.filter((k) => short.includes(k)), ...short.filter((k) => !first.includes(k) && !["split", "dataset"].includes(k))].slice(0, 6); + const pick = [...first.filter((k) => keys.includes(k)), ...short.filter((k) => !first.includes(k) && !["split", "dataset"].includes(k))].slice(0, 6); return pick.length ? pick : keys.slice(0, 4); } const cellOf = (x) => (x == null ? `–` : typeof x === "string" ? esc(x.length > 80 ? x.slice(0, 80) + "…" : x) : typeof x === "object" ? `${Array.isArray(x) ? `${x.length} items` : "{…}"}` : `${esc(String(x))}`); -function openTask(me, i) { +async function openTask(me, i) { const t = me.tasks?.[i]; if (!t) return; + me.selectedTask = i; + const requestKey = me.taskRequest = (me.taskRequest || 0) + 1; me.el.querySelectorAll("[data-task]").forEach((r) => r.classList.toggle("on", +r.dataset.task === i)); const idx = t.task_index ?? t.index ?? me.taskPage.start + i; const d = $("#tk-detail", me.el); d.hidden = false; const hub = me.L.harbor_caps?.datasets.find((x) => x.name === me.taskPage.split)?.hub; const rel = hub && typeof t.task_id === "string" && t.task_id.startsWith(me.taskPage.split + "/") ? t.task_id.slice(me.taskPage.split.length + 1) : null; - d.innerHTML = `
        ${esc(t.task_name || t.task_id || t.id || `Task ${idx}`)} + const title = t.task_name || t.title || t.task_id || t.id || `Task ${idx + 1}`; + const link = new URL(location.href); + link.search = new URLSearchParams({ env: me.taskPage.env, split: me.taskPage.split, task: idx }); + link.hash = "sec-tasks"; + const canPlay = me.L.openenv && !me.L.harbor && me.taskPage.env === me.L.task_api.env; + d.innerHTML = `
        Task ${fmt.format(me.taskPage.start + i + 1)} · ${esc(shortSplit(me.taskPage.split))}${esc(title)}
        + + +
        +
        ${copyBtn(link.href, "Copy task link")} ${rel ? `${icon("database", 13)}Open in the explorer` : ""} - ${me.L.openenv && !me.L.harbor ? `` : ""} -
        -
        ${fieldsView(t)}
        `; + ${canPlay ? `` : ""}
        +
        ${taskContent(t)}

        Loading the full task…

        `; d.scrollIntoView({ block: "nearest", behavior: "smooth" }); + try { + const full = await api(`/api/spaces/${enc(me.spec)}/task?${new URLSearchParams({ env: me.taskPage.env, split: me.taskPage.split, index: idx })}`); + if (page !== me || requestKey !== me.taskRequest || me.selectedTask !== i || !d.isConnected || d.hidden) return; + if (full.task && typeof full.task === "object" && !Array.isArray(full.task)) { + $(".tk-detail-b", d).innerHTML = taskContent(full.task); + } else $("#tk-full-status", d).textContent = "Showing the task record returned by the list endpoint."; + } catch (error) { + if (page === me && requestKey === me.taskRequest && me.selectedTask === i && d.isConnected && !d.hidden) + $("#tk-full-status", d).textContent = `Showing the list record. Full task unavailable: ${error.message}`; + } } - diff --git a/web/js/support.js b/web/js/support.js new file mode 100644 index 0000000000000000000000000000000000000000..876547785619838d53847dfd4a01ac729a4050dc --- /dev/null +++ b/web/js/support.js @@ -0,0 +1,28 @@ +// One capability presentation for dataset readers and live environment transports. +import { esc } from "./util.js"; +import { icon } from "./icons.js"; + +const STATES = { + available: ["Available here", "check"], + external: ["Use native framework", "external"], + unavailable: ["Unavailable here", "info"], + unknown: ["Check availability", "info"], +}; + +export function supportPanel(s) { + if (!s?.capabilities?.length) return ""; + const f = s.framework || {}; + const docs = /^https:\/\//.test(f.docs || "") ? f.docs : null; + return `
        +

        What you can do here

        +
        ${s.capabilities.map((c) => { + const [status, glyph] = STATES[c.state] || STATES.unknown; + return `
        + ${esc(c.label)}${icon(glyph, 13)}${esc(status)}

        ${esc(c.detail)}

        `; + }).join("")}
        +
        How ${esc(f.label || "this environment")} works +

        ${esc(f.lifecycle)}

        ${esc(f.reward)}

        + ${s.transport ? `

        Connection: ${esc(s.transport)}

        ` : ""} + ${docs ? `${esc(f.label)} documentation ${icon("external", 12)}` : ""} +
        `; +} diff --git a/web/js/task.js b/web/js/task.js index dfd39e951d6cbdcea44295c42457c1a36539cd51..34e120330a11377775e97b5ef80744057c586139 100644 --- a/web/js/task.js +++ b/web/js/task.js @@ -11,6 +11,8 @@ import { picker } from "./picker.js"; import { fileViewer } from "./files.js"; import { blocks, inline, loadRenderers } from "./blocks.js"; import { typeset } from "./math.js"; +import { supportPanel } from "./support.js"; +import { rewardDistribution } from "./reward-stats.js"; let models = null; const getModels = () => (models ||= api("/api/models").catch((e) => { models = null; throw e; })); @@ -36,7 +38,6 @@ export async function mount(el, { spec, path: ref, qs }) {
        ${panel([96, 100, 88, 94, 70, 92, 60])}${panel([90, 70, 84])}${panel([80, 60])}
        `; - getModels().catch(() => {}); // start fetching models while the page loads let v, renderers; try { v = await api(`${envApi(spec)}/task?ref=${encodeURIComponent(ref)}`, { retry: 2 }); @@ -46,6 +47,7 @@ export async function mount(el, { spec, path: ref, qs }) { `${icon("arrowLeft", 15)}Open the environment`)}
        `; return; } + if (!el.isConnected) return; if (v.ref !== ref) { // an older address of this task (a row number): the page lives at its own ref ref = v.ref; history.replaceState(null, "", `${taskHref(spec, ref)}${qs.toString() ? `?${qs}` : ""}`); @@ -83,6 +85,7 @@ export async function mount(el, { spec, path: ref, qs }) { ${same.length || related.length ? `
        ${[...same, ...related].map((l) => `

        ${icon(l.rel === "same" ? "link" : "external", 13)} ${esc(l.label)}${l.note ? ` ${esc(l.note)}` : ""}

        `).join("")}
        ` : ""} + ${supportPanel(v.support)}
        ${sections.map((s) => `
        @@ -203,7 +206,10 @@ export async function runCard(box, v, spec, ref) { return; } if (!o.ok) { - box.innerHTML = `${head}
        ${choose()}${about(o)}
        ${icon("alert")}This task can't run here: ${esc(o.why || "no runnable form")}.
        `; + const external = v.support?.capabilities?.some((c) => c.id === "run" && c.state === "external"); + const guide = v.sections?.find((sec) => sec.id === "run"); + box.innerHTML = `${head}
        ${choose()}${about(o)}
        ${icon(external ? "info" : "alert")}${esc(o.why || "No runnable form is available here.")}
        + ${guide ? `` : ""}
        `; return; } if (!s.user) { @@ -405,13 +411,12 @@ function fieldValues(box, fields, pickers) { // Every public rollout of it (how rewards spread, how each model does) and yours; tick two or more to compare them. function stats(runs) { const scored = runs.filter((r) => r.reward != null && r.status === "done"); - const hist = Array(10).fill(0); - scored.forEach((r) => (hist[Math.min(9, Math.floor(r.reward * 10))] += 1)); + const distribution = rewardDistribution(scored.map((r) => r.reward)); const by = new Map(); scored.forEach((r) => { const k = r.endpoint ? `${r.model} (own endpoint)` : r.model || "?"; by.set(k, [...(by.get(k) || []), r.reward]); }); const models = [...by].map(([m, xs]) => ({ model: m, runs: xs.length, mean: xs.reduce((a, b) => a + b, 0) / xs.length, best: Math.max(...xs) })) .sort((a, b) => b.mean - a.mean || b.runs - a.runs); - return { runs: runs.length, scored: scored.length, hist, models, full: scored.filter((r) => r.reward >= 0.999).length, + return { runs: runs.length, scored: scored.length, ...distribution, models, mean: scored.length ? scored.reduce((a, r) => a + r.reward, 0) / scored.length : null }; } @@ -435,16 +440,16 @@ async function taskRuns(box, spec, ref, v, sync) { box.innerHTML = `${s.scored ? `
        Public rollouts${s.runs}
        Mean reward${s.mean == null ? "–" : s.mean.toFixed(2)}
        -
        Full marks${s.full} of ${s.scored}
        +
        Scored rollouts${s.scored}
        Models tried${s.models.length}
        Reward spread
        -
        ${s.hist.map((n, i) => ``).join("")}
        -
        00.51
        +
        ${s.hist.map((n, i) => ``).join("")}
        +
        ${rewardText(s.min)}${rewardText((s.min + s.max) / 2)}${rewardText(s.max)}
        By model
        ${s.models.slice(0, 8).map((m) => ` - `).join("")}
        ModelRunsMeanBest
        ${esc(m.model.split("/")[1] || m.model)}${m.runs}${m.mean.toFixed(2)}${rewardText(m.best)}
        + ${m.mean.toFixed(2)}${rewardText(m.best)}`).join("")}
        ` : ""}
        Rollouts${all.length > 1 ? 'tick two or more to compare them' : ""}
        ${all.map((r) => `