| { |
| "aggregate_results_path": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/runs/step10000/FrankaJointPosVlaHarnessEvalConfig/external-eval-lr_custom_16k-step10000/aggregate_results.json", |
| "benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark", |
| "benchmark_name": null, |
| "checkpoint_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000-unsharded", |
| "checkpoint_format": "molmobot_unsharded", |
| "checkpoint_path": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000-unsharded", |
| "duration_seconds": 2016.921890544938, |
| "episode_count": 40, |
| "eval_kind": "intermediate", |
| "eval_returncode": 0, |
| "final_eval_index": null, |
| "final_eval_label": null, |
| "final_eval_name": null, |
| "final_eval_wandb_prefix": null, |
| "finished_at": "2026-07-17T12:45:30.410588+00:00", |
| "logs": { |
| "eval_driver": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/logs/step10000/eval_driver.log", |
| "model_server": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/logs/step10000/model_server.log" |
| }, |
| "metrics": { |
| "avg_episode_length": 274.675, |
| "avg_successful_episode_length": 140.08, |
| "completed_episode_count": 40, |
| "coverage_adjusted_success_rate": 0.625, |
| "duplicate_episode_count": 0, |
| "episode_count": 40, |
| "expected_episode_count": 40, |
| "failed_shard_count": 0, |
| "failure_count": 15, |
| "house_success_rates": { |
| "house_1088": 0.625 |
| }, |
| "lax_failure_count": 14, |
| "lax_house_success_rates": { |
| "house_1088": 0.65 |
| }, |
| "lax_oracle_done_count": 27, |
| "lax_oracle_done_rate": 0.675, |
| "lax_oracle_done_total_episodes": 40, |
| "lax_success_count": 26, |
| "lax_success_rate": 0.65, |
| "lax_total_episodes": 40, |
| "max_episode_length": 499, |
| "min_episode_length": 32, |
| "missing_episode_count": 0, |
| "num_houses": 1, |
| "oracle_done_count": 25, |
| "oracle_done_rate": 0.625, |
| "success_count": 25, |
| "success_rate": 0.625, |
| "total_episodes": 40, |
| "unexpected_result_count": 0 |
| }, |
| "output_root": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/runs/step10000", |
| "request": { |
| "checkpoint_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000-unsharded", |
| "created_at": "2026-07-17T12:11:48.152584+00:00", |
| "external_eval": { |
| "benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark", |
| "camera_names": "droid_shoulder_light_randomization wrist_camera_zed_mini", |
| "coordinator_poll_interval_seconds": 30, |
| "enabled": true, |
| "eval_config_cls": "scripts.eval.vla_eval_harness_adapter:FrankaJointPosVlaHarnessEvalConfig", |
| "eval_extra_args": "--task_horizon_steps 500", |
| "eval_renderer": "filament", |
| "eval_wandb_enabled": true, |
| "eval_wandb_group": "lr-custom-ie-16k-b16-32", |
| "eval_wandb_project": "mlspaces-json-eval", |
| "eval_wandb_tags": "betty,vla-harness,molmospaces,intermediate-eval,molmobot-droid,left-right-pick-place,simple-desk,relative-next-to,sharded-intermediate-eval,filament,10x", |
| "final": { |
| "benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark", |
| "benchmarks": [ |
| { |
| "benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark", |
| "camera_names": null, |
| "eval_config_cls": null, |
| "eval_extra_args": "--task_horizon_steps 500", |
| "eval_renderer": null, |
| "model_server_extra_args": null, |
| "name": "FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench", |
| "num_workers": null, |
| "parallel_shards": null, |
| "prompt_variant": null |
| } |
| ], |
| "camera_names": "droid_shoulder_light_randomization wrist_camera_zed_mini", |
| "coordinator_poll_interval_seconds": 30, |
| "enabled": true, |
| "eval_config_cls": "scripts.eval.vla_eval_harness_adapter:FrankaJointPosVlaHarnessEvalConfig", |
| "eval_extra_args": "--task_horizon_steps 500", |
| "eval_renderer": "filament", |
| "eval_wandb_enabled": true, |
| "eval_wandb_group": "lr-custom-fe-16k-b16-32", |
| "eval_wandb_project": "mlspaces-json-eval", |
| "eval_wandb_tags": "betty,vla-harness,molmospaces,final-eval,molmobot-droid,left-right-pick-place,simple-desk,relative-next-to,sharded-final-eval,filament,10x", |
| "log_root": null, |
| "model_server_extra_args": "--args.execute_horizon 8 --args.states_mode cross_attn --args.max_joint_delta 0.2", |
| "model_server_python": null, |
| "model_server_pythonpath": null, |
| "model_server_script": null, |
| "molmobot_root": "/vast/projects/dineshj/lab/ccwatson/Documents/MolmoBot/MolmoBot", |
| "molmospaces_python": null, |
| "molmospaces_root": "/vast/projects/dineshj/lab/ccwatson/Documents/molmospaces", |
| "num_workers": 1, |
| "parallel_shards": 10, |
| "prompt_variant": "natural_language", |
| "repo_root": "/vast/projects/dineshj/lab/ccwatson/Documents/MolmoBot", |
| "request_dir": null, |
| "required": true, |
| "result_dir": null, |
| "run_root": null, |
| "server_port": null, |
| "server_port_base": 38000, |
| "server_port_span": 20000, |
| "upload_logs": true, |
| "vla_harness_root": "/vast/projects/dineshj/lab/ccwatson/Documents/vla-evaluation-harness", |
| "wait_seconds": 7200 |
| }, |
| "final_result_wait_seconds": 7200, |
| "interval": 2000, |
| "log_root": null, |
| "metric_for_best": "success_rate", |
| "metric_mode": "max", |
| "model_server_extra_args": "--args.execute_horizon 8 --args.states_mode cross_attn --args.max_joint_delta 0.2", |
| "model_server_python": null, |
| "model_server_pythonpath": null, |
| "num_checkpoints_to_keep": 2, |
| "num_workers": 1, |
| "parallel_shards": 10, |
| "prompt_variant": "natural_language", |
| "request_dir": null, |
| "result_dir": null, |
| "result_poll_interval_steps": 50, |
| "run_root": null, |
| "wait_for_results": false |
| }, |
| "run_name": "lr_custom_16k", |
| "save_folder": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k", |
| "schema_version": 1, |
| "sharded_checkpoint_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000", |
| "step": 10000, |
| "training_wandb": { |
| "id": "ouuzdjlg", |
| "url": "https://wandb.ai/ccwatson/molmobot_finetune/runs/ouuzdjlg" |
| } |
| }, |
| "resolved_output_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/runs/step10000/FrankaJointPosVlaHarnessEvalConfig/external-eval-lr_custom_16k-step10000", |
| "schema_version": 1, |
| "server": { |
| "host": "127.0.0.1", |
| "port": 29253, |
| "port_base": 18000, |
| "port_span": 20000 |
| }, |
| "started_at": "2026-07-17T12:11:53.488701+00:00", |
| "status": "succeeded", |
| "step": 10000, |
| "summary_source": "aggregate_results_json", |
| "wandb": { |
| "enabled": true, |
| "group": "lr-custom-ie-16k-b16-32", |
| "name": "lr_custom_16k external eval step 10000", |
| "project": "mlspaces-json-eval", |
| "run_id": "external-eval-lr_custom_16k-step10000", |
| "tags": "betty,vla-harness,molmospaces,intermediate-eval,molmobot-droid,left-right-pick-place,simple-desk,relative-next-to,sharded-intermediate-eval,filament,10x" |
| } |
| } |
|
|
