| """Evidence-based capabilities shared by dataset adapters, live Spaces, UI and MCP. |
| |
| A framework label describes a format, not permission or proof that it can run. |
| A dataset's native harness may be external; a live server may be asleep or may |
| speak HTTP instead of OpenEnv's WebSocket protocol. Keep those axes independent. |
| The contract is additive so older clients can keep using run.options/step_api. |
| """ |
| from __future__ import annotations |
|
|
| from dataclasses import asdict, dataclass |
| from typing import Any |
|
|
|
|
| @dataclass(frozen=True) |
| class Framework: |
| id: str |
| label: str |
| task: str |
| lifecycle: str |
| reward: str |
| docs: str |
|
|
|
|
| FRAMEWORKS = { |
| f.id: f for f in ( |
| Framework("harbor", "Harbor", "Task folder or packed task files", |
| "Prepare sandbox → run agent → verify → collect artifacts", |
| "The task's verifier writes a scalar or named reward metrics.", |
| "https://www.harborframework.com/docs/tasks"), |
| Framework("openenv", "OpenEnv", "Live environment with typed actions and observations", |
| "Open session → reset → step until done → close", |
| "The environment returns rewards and termination with observations.", |
| "https://huggingface.co/docs/openenv/main/reference/core"), |
| Framework("verifiers", "Verifiers", "Rows loaded by an installed environment or taskset", |
| "Load environment/taskset → run its harness → score the rollout", |
| "The installed environment's rubric or judge defines the reward; rows alone do not include that runtime.", |
| "https://github.com/PrimeIntellect-ai/verifiers/tree/main/docs"), |
| Framework("nemo-gym", "NeMo Gym", "Task data plus agent and resources-server configuration", |
| "Seed → interact → verify, or reset → step for Gymnasium servers", |
| "A resources server or embedded agent harness computes the reward.", |
| "https://docs.nvidia.com/nemo/gym/build-verifiers/"), |
| Framework("verl", "verl", "Training rows with prompt and reward metadata", |
| "Generate responses → route to the configured reward function → train", |
| "data_source selects a scorer unless training config supplies a custom reward function.", |
| "https://verl.readthedocs.io/en/latest/advance/reward_loop.html"), |
| Framework("skyrl", "SkyRL", "Training rows routed to a skyrl-gym environment", |
| "Load rows → run environment interactions → compute training rewards", |
| "The selected environment and its configuration define the reward.", |
| "https://skyrl.readthedocs.io/en/latest/"), |
| Framework("mimo", "MiMo", "Tasks from the MiMo RL environment release", |
| "Prepare task → run the release's domain harness → grade", |
| "Each domain uses its own verifier; supported conversions also run through Harbor.", |
| "https://huggingface.co/datasets/XiaomiMiMo/MiMo-V2.6-RL-oss"), |
| Framework("gymnasium", "Gymnasium HTTP", "A server exposing reset and step over HTTP", |
| "Reset → step until terminated or truncated → close", |
| "The server returns step rewards; its HTTP schema defines the request and response.", |
| "https://gymnasium.farama.org/api/env/"), |
| Framework("mcp", "MCP", "Tools exposed by a live server", "Connect → call advertised tools → close", |
| "A reward is available only when the server exposes a grader or returns one.", |
| "https://modelcontextprotocol.io/specification/2025-06-18/server/tools"), |
| Framework("ors", "ORS", "A server with named environments and sessions", |
| "Create session → call environment → close session", "The server defines its grading contract.", ""), |
| Framework("api", "HTTP API", "A live server's published operations", "Call the server's documented routes", |
| "Inspect the published schema and grading routes for this server.", ""), |
| Framework("rows", "RL dataset", "Dataset rows", "Browse task data; use the source's environment to execute it", |
| "A dataset label or answer column alone does not establish a runnable grader.", ""), |
| ) |
| } |
| HOSTED_RUNNERS = frozenset({"harbor", "mimo", "nemo-gym"}) |
|
|
|
|
| def framework(value: str | None) -> Framework: |
| key = str(value or "").lower().replace("_", "-").replace(" ", "-") |
| aliases = {"nemo": "nemo-gym", "nemogym": "nemo-gym", "generic": "rows", "rl": "rows", "harbor-(packed)": "harbor"} |
| key = aliases.get(key, key) |
| if key.startswith("mimo"): |
| key = "mimo" |
| return FRAMEWORKS.get(key, FRAMEWORKS["rows"]) |
|
|
|
|
| def capability(id: str, label: str, state: str, detail: str) -> dict[str, str]: |
| if state not in {"available", "external", "unavailable", "unknown"}: |
| raise ValueError(f"invalid capability state: {state}") |
| return {"id": id, "label": label, "state": state, "detail": detail} |
|
|
|
|
| def dataset_support(name: str, *, state: str = "ready", options: list[dict[str, Any]] | None = None, |
| note: str = "", revision: str | None = None, restricted: bool = False) -> dict[str, Any]: |
| f = framework(name) |
| ready = state == "ready" |
| features = [capability("browse", "Browse tasks", "available" if ready else "unknown", |
| "Inspect the prompt, grading information and source files." if ready else "Task discovery is still in progress.")] |
| if options is None: |
| features.append(capability("run", "Run an agent", "unknown", "Open a task to check its harness and runtime requirements.")) |
| else: |
| runnable = [o for o in options if o.get("ok") and o.get("runner") in HOSTED_RUNNERS and not o.get("href")] |
| external = [o for o in options if o.get("href") or (not o.get("ok") and o.get("runner") in {"verifiers", "nemo-gym", "nemogym", "verl", "skyrl"})] |
| why = "; ".join(dict.fromkeys(o.get("why") for o in options if o.get("why"))) |
| features.append(capability("run", "Run an agent", "available" if runnable else "external" if external else "unavailable", |
| "Runs here with " + ", ".join(o["label"] for o in runnable) + "." if runnable else |
| why or note or "This source has no executable harness configured here.")) |
| features.append(capability("mcp", "Explore with an agent", "unavailable" if restricted else "available" if ready else "unknown", |
| "The MCP bridge only serves public datasets; this source requires your sign-in." if restricted else |
| "MCP exposes public task browsing. Execution follows the runtime support above.")) |
| return {"version": 1, "framework": asdict(f), "source": "dataset", "evidence": "adapter", "revision": revision, |
| "transport": None, "capabilities": features} |
|
|
|
|
| def space_support(info: dict[str, Any]) -> dict[str, Any]: |
| up = bool(info.get("running")) |
| f = framework(info.get("framework") or "api") |
| transport = "openenv-ws" if info.get("openenv") else "http" if info.get("api") else "mcp-http" if info.get("mcp") else None |
| can_play = bool(info.get("step_api") or info.get("mcp") or info.get("api")) |
| status = "available" if up and can_play else "unavailable" if up else "unknown" |
| detail = ("Play an episode over OpenEnv's WebSocket session." if transport == "openenv-ws" else |
| "Call the published HTTP routes; each explorer session keeps its own cookies." if transport == "http" else |
| "Call MCP tools. Episode state depends on the server's MCP implementation.") if can_play else "No supported interactive API was discovered." |
| if not up: |
| detail = "The Space is " + str(info.get("stage") or "unreachable").lower().replace("_", " ") + "; check its live interface when it is running." |
| return {"version": 1, "framework": asdict(f), "source": "space", "evidence": "live" if up else "unverified", |
| "transport": transport, "capabilities": [ |
| capability("play", "Interactive playground", status, detail), |
| capability("browse", "Browse tasks", "available" if up and info.get("task_api") else "unavailable" if up else "unknown", |
| "Tasks are served by the environment's Task API." if info.get("task_api") else "No live Task API was discovered; source files may describe the tasks."), |
| capability("mcp", "Connect an agent", status, |
| "The explorer bridges the advertised interface to MCP." if up and can_play else detail), |
| ]} |
|
|
