File size: 8,459 Bytes
5c0ec0f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | {
"aggregate_results_path": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/runs/step10000/FrankaJointPosVlaHarnessEvalConfig/external-eval-lr_custom_16k-step10000/aggregate_results.json",
"benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark",
"benchmark_name": null,
"checkpoint_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000-unsharded",
"checkpoint_format": "molmobot_unsharded",
"checkpoint_path": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000-unsharded",
"duration_seconds": 2016.921890544938,
"episode_count": 40,
"eval_kind": "intermediate",
"eval_returncode": 0,
"final_eval_index": null,
"final_eval_label": null,
"final_eval_name": null,
"final_eval_wandb_prefix": null,
"finished_at": "2026-07-17T12:45:30.410588+00:00",
"logs": {
"eval_driver": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/logs/step10000/eval_driver.log",
"model_server": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/logs/step10000/model_server.log"
},
"metrics": {
"avg_episode_length": 274.675,
"avg_successful_episode_length": 140.08,
"completed_episode_count": 40,
"coverage_adjusted_success_rate": 0.625,
"duplicate_episode_count": 0,
"episode_count": 40,
"expected_episode_count": 40,
"failed_shard_count": 0,
"failure_count": 15,
"house_success_rates": {
"house_1088": 0.625
},
"lax_failure_count": 14,
"lax_house_success_rates": {
"house_1088": 0.65
},
"lax_oracle_done_count": 27,
"lax_oracle_done_rate": 0.675,
"lax_oracle_done_total_episodes": 40,
"lax_success_count": 26,
"lax_success_rate": 0.65,
"lax_total_episodes": 40,
"max_episode_length": 499,
"min_episode_length": 32,
"missing_episode_count": 0,
"num_houses": 1,
"oracle_done_count": 25,
"oracle_done_rate": 0.625,
"success_count": 25,
"success_rate": 0.625,
"total_episodes": 40,
"unexpected_result_count": 0
},
"output_root": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/runs/step10000",
"request": {
"checkpoint_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000-unsharded",
"created_at": "2026-07-17T12:11:48.152584+00:00",
"external_eval": {
"benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark",
"camera_names": "droid_shoulder_light_randomization wrist_camera_zed_mini",
"coordinator_poll_interval_seconds": 30,
"enabled": true,
"eval_config_cls": "scripts.eval.vla_eval_harness_adapter:FrankaJointPosVlaHarnessEvalConfig",
"eval_extra_args": "--task_horizon_steps 500",
"eval_renderer": "filament",
"eval_wandb_enabled": true,
"eval_wandb_group": "lr-custom-ie-16k-b16-32",
"eval_wandb_project": "mlspaces-json-eval",
"eval_wandb_tags": "betty,vla-harness,molmospaces,intermediate-eval,molmobot-droid,left-right-pick-place,simple-desk,relative-next-to,sharded-intermediate-eval,filament,10x",
"final": {
"benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark",
"benchmarks": [
{
"benchmark_dir": "/vast/projects/dineshj/lab/ccwatson/Documents/molmo-robotics-stack/components/molmospaces/benchmarks/custom/molmospaces-bench-v2/procthor-objaverse/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench/FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench_20260716_json_benchmark",
"camera_names": null,
"eval_config_cls": null,
"eval_extra_args": "--task_horizon_steps 500",
"eval_renderer": null,
"model_server_extra_args": null,
"name": "FrankaPickAndPlaceRelativeNextToSimpleDeskDiverseTargetsTested40Bench",
"num_workers": null,
"parallel_shards": null,
"prompt_variant": null
}
],
"camera_names": "droid_shoulder_light_randomization wrist_camera_zed_mini",
"coordinator_poll_interval_seconds": 30,
"enabled": true,
"eval_config_cls": "scripts.eval.vla_eval_harness_adapter:FrankaJointPosVlaHarnessEvalConfig",
"eval_extra_args": "--task_horizon_steps 500",
"eval_renderer": "filament",
"eval_wandb_enabled": true,
"eval_wandb_group": "lr-custom-fe-16k-b16-32",
"eval_wandb_project": "mlspaces-json-eval",
"eval_wandb_tags": "betty,vla-harness,molmospaces,final-eval,molmobot-droid,left-right-pick-place,simple-desk,relative-next-to,sharded-final-eval,filament,10x",
"log_root": null,
"model_server_extra_args": "--args.execute_horizon 8 --args.states_mode cross_attn --args.max_joint_delta 0.2",
"model_server_python": null,
"model_server_pythonpath": null,
"model_server_script": null,
"molmobot_root": "/vast/projects/dineshj/lab/ccwatson/Documents/MolmoBot/MolmoBot",
"molmospaces_python": null,
"molmospaces_root": "/vast/projects/dineshj/lab/ccwatson/Documents/molmospaces",
"num_workers": 1,
"parallel_shards": 10,
"prompt_variant": "natural_language",
"repo_root": "/vast/projects/dineshj/lab/ccwatson/Documents/MolmoBot",
"request_dir": null,
"required": true,
"result_dir": null,
"run_root": null,
"server_port": null,
"server_port_base": 38000,
"server_port_span": 20000,
"upload_logs": true,
"vla_harness_root": "/vast/projects/dineshj/lab/ccwatson/Documents/vla-evaluation-harness",
"wait_seconds": 7200
},
"final_result_wait_seconds": 7200,
"interval": 2000,
"log_root": null,
"metric_for_best": "success_rate",
"metric_mode": "max",
"model_server_extra_args": "--args.execute_horizon 8 --args.states_mode cross_attn --args.max_joint_delta 0.2",
"model_server_python": null,
"model_server_pythonpath": null,
"num_checkpoints_to_keep": 2,
"num_workers": 1,
"parallel_shards": 10,
"prompt_variant": "natural_language",
"request_dir": null,
"result_dir": null,
"result_poll_interval_steps": 50,
"run_root": null,
"wait_for_results": false
},
"run_name": "lr_custom_16k",
"save_folder": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k",
"schema_version": 1,
"sharded_checkpoint_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/step10000",
"step": 10000,
"training_wandb": {
"id": "ouuzdjlg",
"url": "https://wandb.ai/ccwatson/molmobot_finetune/runs/ouuzdjlg"
}
},
"resolved_output_dir": "/ceph/projects/dineshj/lab/ccwatson/MolmoBot_runs/lr_custom_16k/external_eval/runs/step10000/FrankaJointPosVlaHarnessEvalConfig/external-eval-lr_custom_16k-step10000",
"schema_version": 1,
"server": {
"host": "127.0.0.1",
"port": 29253,
"port_base": 18000,
"port_span": 20000
},
"started_at": "2026-07-17T12:11:53.488701+00:00",
"status": "succeeded",
"step": 10000,
"summary_source": "aggregate_results_json",
"wandb": {
"enabled": true,
"group": "lr-custom-ie-16k-b16-32",
"name": "lr_custom_16k external eval step 10000",
"project": "mlspaces-json-eval",
"run_id": "external-eval-lr_custom_16k-step10000",
"tags": "betty,vla-harness,molmospaces,intermediate-eval,molmobot-droid,left-right-pick-place,simple-desk,relative-next-to,sharded-intermediate-eval,filament,10x"
}
}
|