{ "aggregate_results_path": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/runs/step10000/FrankaJointPosVlaHarnessEvalConfig/external-eval-lr_custom_16k-step10000/aggregate_results.json", "benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark", "benchmark_name": null, "checkpoint_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000-unsharded", "checkpoint_format": "molmobot_unsharded", "checkpoint_path": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000-unsharded", "duration_seconds": 2016.921890544938, "episode_count": 40, "eval_kind": "intermediate", "eval_returncode": 0, "final_eval_index": null, "final_eval_label": null, "final_eval_name": null, "final_eval_wandb_prefix": null, "finished_at": "2026-07-17T12:45:30.410588+00:00", "logs": { "eval_driver": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/logs/step10000/eval_driver.log", "model_server": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/logs/step10000/model_server.log" }, "metrics": { "avg_episode_length": 274.675, "avg_successful_episode_length": 140.08, "completed_episode_count": 40, "coverage_adjusted_success_rate": 0.625, "duplicate_episode_count": 0, "episode_count": 40, "expected_episode_count": 40, "failed_shard_count": 0, "failure_count": 15, "house_success_rates": { "house_1088": 0.625 }, "lax_failure_count": 14, "lax_house_success_rates": { "house_1088": 0.65 }, "lax_oracle_done_count": 27, "lax_oracle_done_rate": 0.675, "lax_oracle_done_total_episodes": 40, "lax_success_count": 26, "lax_success_rate": 0.65, "lax_total_episodes": 40, "max_episode_length": 499, "min_episode_length": 32, "missing_episode_count": 0, "num_houses": 1, "oracle_done_count": 25, "oracle_done_rate": 0.625, "success_count": 25, "success_rate": 0.625, "total_episodes": 40, "unexpected_result_count": 0 }, "output_root": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/runs/step10000", "request": { "checkpoint_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000-unsharded", "created_at": "2026-07-17T12:11:48.152584+00:00", "external_eval": { "benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark", "camera_names": "droid_shoulder_light_randomization wrist_camera_zed_mini", "coordinator_poll_interval_seconds": 30, "enabled": true, "eval_config_cls": "scripts.eval.vla_eval_harness_adapter:FrankaJointPosVlaHarnessEvalConfig", "eval_extra_args": "--task_horizon_steps 500", "eval_renderer": "filament", "eval_wandb_enabled": true, "eval_wandb_group": "lr-custom-ie-16k-b16-32", "eval_wandb_project": "mlspaces-json-eval", "eval_wandb_tags": "betty,vla-harness,molmospaces,intermediate-eval,molmobot-droid,left-right-pick-place,simple-desk,relative-next-to,sharded-intermediate-eval,filament,10x", "final": { "benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark", "benchmarks": [ { "benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark", "camera_names": null, "eval_config_cls": null, "eval_extra_args": "--task_horizon_steps 500", "eval_renderer": null, "model_server_extra_args": null, "name": "FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench", "num_workers": null, "parallel_shards": null, "prompt_variant": null } ], "camera_names": "droid_shoulder_light_randomization wrist_camera_zed_mini", "coordinator_poll_interval_seconds": 30, "enabled": true, "eval_config_cls": "scripts.eval.vla_eval_harness_adapter:FrankaJointPosVlaHarnessEvalConfig", "eval_extra_args": "--task_horizon_steps 500", "eval_renderer": "filament", "eval_wandb_enabled": true, "eval_wandb_group": "lr-custom-fe-16k-b16-32", "eval_wandb_project": "mlspaces-json-eval", "eval_wandb_tags": "betty,vla-harness,molmospaces,final-eval,molmobot-droid,left-right-pick-place,simple-desk,relative-next-to,sharded-final-eval,filament,10x", "log_root": null, "model_server_extra_args": "--args.execute_horizon 8 --args.states_mode cross_attn --args.max_joint_delta 0.2", "model_server_python": null, "model_server_pythonpath": null, "model_server_script": null, "molmobot_root": "/vast/projects/dineshj/lab/ccwatson/Documents/MolmoBot/MolmoBot", "molmospaces_python": null, "molmospaces_root": "/vast/projects/dineshj/lab/ccwatson/Documents/molmospaces", "num_workers": 1, "parallel_shards": 10, "prompt_variant": "natural_language", "repo_root": "/vast/projects/dineshj/lab/ccwatson/Documents/MolmoBot", "request_dir": null, "required": true, "result_dir": null, "run_root": null, "server_port": null, "server_port_base": 38000, "server_port_span": 20000, "upload_logs": true, "vla_harness_root": "/vast/projects/dineshj/lab/ccwatson/Documents/vla-evaluation-harness", "wait_seconds": 7200 }, "final_result_wait_seconds": 7200, "interval": 2000, "log_root": null, "metric_for_best": "success_rate", "metric_mode": "max", "model_server_extra_args": "--args.execute_horizon 8 --args.states_mode cross_attn --args.max_joint_delta 0.2", "model_server_python": null, "model_server_pythonpath": null, "num_checkpoints_to_keep": 2, "num_workers": 1, "parallel_shards": 10, "prompt_variant": "natural_language", "request_dir": null, "result_dir": null, "result_poll_interval_steps": 50, "run_root": null, "wait_for_results": false }, "run_name": "lr_custom_16k", "save_folder": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k", "schema_version": 1, "sharded_checkpoint_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000", "step": 10000, "training_wandb": { "id": "ouuzdjlg", "url": "https://wandb.ai/ccwatson/molmobot_finetune/runs/ouuzdjlg" } }, "resolved_output_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/runs/step10000/FrankaJointPosVlaHarnessEvalConfig/external-eval-lr_custom_16k-step10000", "schema_version": 1, "server": { "host": "127.0.0.1", "port": 29253, "port_base": 18000, "port_span": 20000 }, "started_at": "2026-07-17T12:11:53.488701+00:00", "status": "succeeded", "step": 10000, "summary_source": "aggregate_results_json", "wandb": { "enabled": true, "group": "lr-custom-ie-16k-b16-32", "name": "lr_custom_16k external eval step 10000", "project": "mlspaces-json-eval", "run_id": "external-eval-lr_custom_16k-step10000", "tags": "betty,vla-harness,molmospaces,intermediate-eval,molmobot-droid,left-right-pick-place,simple-desk,relative-next-to,sharded-intermediate-eval,filament,10x" } }