eval/aloha_transfer_cube.json

{
  "schema_version": "0.1",
  "source": {
    "paper": "https://arxiv.org/abs/2304.13705",
    "arxiv": "https://arxiv.org/abs/2304.13705",
    "model_variant": "act",
    "evaluated_by": "OpenRAL:openral benchmark run",
    "reproduced_locally": true,
    "reproduction_planned": null,
    "reproduction_cli": "openral benchmark run --suite aloha_transfer_cube --rskill rskill://act-aloha",
    "table": null,
    "status": "reproduced"
  },
  "benchmark": {
    "name": "ALOHA bimanual cube transfer (gym-aloha)",
    "dataset": null,
    "protocol": "50 episodes per task, success_key=is_success, max_steps=400",
    "robot": "aloha_bimanual",
    "simulator": "gym-aloha (MuJoCo)"
  },
  "eval_config": {
    "n_episodes": 50,
    "seeds": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17,
      18,
      19,
      20,
      21,
      22,
      23,
      24,
      25,
      26,
      27,
      28,
      29,
      30,
      31,
      32,
      33,
      34,
      35,
      36,
      37,
      38,
      39,
      40,
      41,
      42,
      43,
      44,
      45,
      46,
      47,
      48,
      49
    ],
    "success_key": "is_success",
    "max_steps": 400,
    "vla_id": "act",
    "weights_uri": "rskill://rskills/act-aloha"
  },
  "results": {
    "aloha_transfer_cube/0_success_rate": 0.82,
    "avg_success_rate": 0.82,
    "n_tasks": 1,
    "n_episodes_per_task": 50,
    "n_episodes_total": 50,
    "mean_step_latency_ms_avg": 10.794657473535338
  },
  "baselines": {}
}