{ "_name_or_path": "meta-llama/Llama-3.2-11B-Vision-Instruct/custom", "architectures": [ "MllamaForConditionalGenerationWM" ], "image_token_index": 128256, "model_type": "mllama", "text_config": { "_attn_implementation_autoset": false, "_name_or_path": "", "add_cross_attention": false, "architectures": null, "bad_words_ids": null, "begin_suppress_tokens": null, "bos_token_id": 128000, "chunk_size_feed_forward": 0, "cross_attention_hidden_size": null, "cross_attention_layers": [ 3, 8, 13, 18, 23, 28, 33, 38 ], "decoder_start_token_id": null, "diversity_penalty": 0.0, "do_sample": false, "dropout": 0, "early_stopping": false, "encoder_no_repeat_ngram_size": 0, "eos_token_id": [ 128001, 128008, 128009 ], "exponential_decay_length_penalty": null, "finetuning_task": null, "forced_bos_token_id": null, "forced_eos_token_id": null, "hidden_act": "silu", "hidden_size": 4096, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "initializer_range": 0.02, "intermediate_size": 14336, "is_decoder": false, "is_encoder_decoder": false, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "length_penalty": 1.0, "max_length": 20, "max_position_embeddings": 131072, "min_length": 0, "model_type": "mllama_text_model", "no_repeat_ngram_size": 0, "num_attention_heads": 32, "num_beam_groups": 1, "num_beams": 1, "num_hidden_layers": 40, "num_key_value_heads": 8, "num_return_sequences": 1, "output_attentions": false, "output_hidden_states": false, "output_scores": false, "pad_token_id": 128004, "prefix": null, "problem_type": null, "pruned_heads": {}, "remove_invalid_values": false, "repetition_penalty": 1.0, "return_dict": true, "return_dict_in_generate": false, "rms_norm_eps": 1e-05, "rope_scaling": { "factor": 8.0, "high_freq_factor": 4.0, "low_freq_factor": 1.0, "original_max_position_embeddings": 8192, "rope_type": "llama3" }, "rope_theta": 500000.0, "sep_token_id": null, "suppress_tokens": null, "task_specific_params": null, "temperature": 1.0, "tf_legacy_loss": false, "tie_encoder_decoder": false, "tie_word_embeddings": false, "tokenizer_class": null, "top_k": 50, "top_p": 1.0, "torch_dtype": "bfloat16", "torchscript": false, "typical_p": 1.0, "use_bfloat16": false, "use_cache": true, "vocab_size": 128256 }, "torch_dtype": "bfloat16", "transformers_version": "4.46.2", "vision_config": { "_attn_implementation_autoset": false, "_name_or_path": "", "add_cross_attention": false, "architectures": null, "attention_heads": 16, "bad_words_ids": null, "begin_suppress_tokens": null, "bos_token_id": null, "chunk_size_feed_forward": 0, "cross_attention_hidden_size": null, "decoder_start_token_id": null, "diversity_penalty": 0.0, "do_sample": false, "early_stopping": false, "encoder_no_repeat_ngram_size": 0, "eos_token_id": null, "exponential_decay_length_penalty": null, "finetuning_task": null, "forced_bos_token_id": null, "forced_eos_token_id": null, "hidden_act": "gelu", "hidden_size": 1280, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "image_size": 560, "initializer_range": 0.02, "intermediate_layers_indices": [ 3, 7, 15, 23, 30 ], "intermediate_size": 5120, "is_decoder": false, "is_encoder_decoder": false, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "length_penalty": 1.0, "max_length": 20, "max_num_tiles": 1, "min_length": 0, "model_type": "mllama_vision_model", "no_repeat_ngram_size": 0, "norm_eps": 1e-05, "num_beam_groups": 1, "num_beams": 1, "num_channels": 3, "num_global_layers": 8, "num_hidden_layers": 32, "num_return_sequences": 1, "output_attentions": false, "output_hidden_states": false, "output_scores": false, "pad_token_id": null, "patch_size": 14, "prefix": null, "problem_type": null, "pruned_heads": {}, "remove_invalid_values": false, "repetition_penalty": 1.0, "return_dict": true, "return_dict_in_generate": false, "sep_token_id": null, "supported_aspect_ratios": [ [ 1, 1 ], [ 1, 2 ], [ 1, 3 ], [ 1, 4 ], [ 2, 1 ], [ 2, 2 ], [ 3, 1 ], [ 4, 1 ] ], "suppress_tokens": null, "task_specific_params": null, "temperature": 1.0, "tf_legacy_loss": false, "tie_encoder_decoder": false, "tie_word_embeddings": true, "tokenizer_class": null, "top_k": 50, "top_p": 1.0, "torch_dtype": "bfloat16", "torchscript": false, "typical_p": 1.0, "use_bfloat16": false, "vision_output_dim": 7680 }, "wm_config": { "defaults": { "act": "SiLU", "actType": "Tanh", "action_repeat": 1, "action_space": [ 7 ], "actor": { "dist": "normal", "entropy": 0.0003, "eps": 1e-05, "grad_clip": 100.0, "layers": 2, "lr": 3e-05, "max_std": 1.0, "min_std": 0.1, "outscale": 1.0, "std": "learned", "temp": 0.1, "unimix_ratio": 0.01 }, "actor_only_scale": 0.0, "annealing": null, "architecture": [ 100, 100 ], "augment_images": false, "batch_length": 1500, "batch_size": 1, "bc_loss_only": false, "bc_reg": true, "bc_reg_wd": 1.0, "bc_reg_weight": 1.0, "checkPeriod": 10000, "classifier_batch_length": 1500, "classifier_batch_size": 32, "classifier_epoch": 50, "classifier_latent_mode": "all", "classifier_mode": "split", "classifier_padding_mode": "last", "compile": true, "consRadius": 0.5, "cont_ensemble_size": 1, "cont_ensemble_subsample": 1, "cont_head": { "layers": 2, "loss_scale": 1.0, "outscale": 1.0 }, "contrast_bg": true, "costType": "sparse", "critic": { "dist": "symlog_disc", "eps": 1e-05, "grad_clip": 100.0, "layers": 2, "lr": 3e-05, "outscale": 0.0, "slow_target": true, "slow_target_fraction": 0.02, "slow_target_update": 1 }, "critic_ensemble_size": 1, "critic_ensemble_subsample": 2, "critic_reset_every": 0, "dataset_size": 1000000, "debug": false, "decay_model_lr": false, "decoder": { "act": "SiLU", "cnn_depth": 32, "cnn_keys": ".*image.*", "cnn_sigmoid": false, "image_dist": "mse", "kernel_size": 4, "minres": 4, "mlp_keys": "state", "mlp_layers": 5, "mlp_units": 1024, "norm": true, "outscale": 1.0, "vector_dist": "symlog_mse" }, "deterministic_run": true, "device": "cuda:0", "disag_action_cond": false, "disag_layers": 4, "disag_log": true, "disag_models": 10, "disag_offset": 1, "disag_target": "stoch", "disag_units": 400, "discount": 0.997, "discount_lambda": 0.95, "doneType": "toEnd", "done_mode": 0, "dropout_recur_in_residuals": false, "dropout_recurrent_prob": 0, "dt": 0.05, "dyn_deter": 512, "dyn_discrete": 0, "dyn_hidden": 512, "dyn_mean_act": "none", "dyn_min_std": 0.1, "dyn_rec_depth": 1, "dyn_scale": 0.5, "dyn_std_act": "sigmoid2", "dyn_stoch": 32, "ema_decay": 0.999, "ema_power": 0.8, "encoder": { "act": "SiLU", "cnn_depth": 32, "cnn_keys": ".*image.*", "kernel_size": 4, "minres": 4, "mlp_keys": "state", "mlp_layers": 5, "mlp_units": 1024, "norm": true, "symlog_inputs": true }, "ensemble_residuals": false, "envs": 1, "eval_every": 500, "eval_num_seeds": 50, "eval_per_seed": 1, "eval_state_mean": false, "evaldir": null, "exp_name": "robocasa-pnpctr2sink", "expl_behavior": "greedy", "expl_extr_scale": 0.0, "expl_intr_scale": 1.0, "expl_until": 0, "failure_data": "demo_im128_failure_front_wrist_view_127_1500.hdf5", "freeze_encoder": true, "from_ckpt": "/home/yilin/Projects/failure_detection/logs/dreamer_cont/1018/192534/pretrain_joint.pt", "from_prefill_dir": null, "gamma": 0.9999, "grad_clip": 1000, "grad_heads": [ "decoder", "cont" ], "grayscale": false, "grid_path": "/home/kensuke/latent-safety/logs/grid/LS_BRT_v1_w1.25.npy", "gt_lx": false, "hybrid_critic_fitting": false, "hybrid_training": true, "ignore_base_policy": false, "imag_gradient": "dynamics", "imag_gradient_mix": 0.0, "imag_horizon": 15, "image": null, "image_size": 64, "initial": "learned", "initial_joint_train_steps": 0, "kl_free": 1.0, "learnedDyn": null, "learnedMargin": null, "learningRate": 0.001, "log_every": 100, "log_every_video": 2000, "logdir": "logs/dreamer_cont", "mask_recur": false, "maxUpdates": 400000, "memoryCapacity": 10000, "mode": "RA", "model_lr": 0.0001, "model_only_scale": 0.0, "multi_task_data": false, "name": "world_model_new", "no_joint_steps": false, "norm": true, "numT": 25, "num_actions": 3, "num_exp_trajs": 100, "num_pts": 5000, "nx": 51, "ny": 51, "nz": 51, "obs_keys": [ "robot0_agentview_front_image", "robot0_eye_in_hand_image" ], "obs_lr": 0.001, "obs_r": 0.5, "obs_step": 1, "obs_x": 0, "obs_y": 0, "observation_space": { "discount": [ 1 ], "privileged_state": [ 3 ], "robot0_agentview_front_image": [ 64, 64, 3 ], "robot0_eef_pos": [ 3 ], "robot0_eef_quat": [ 4 ], "robot0_eye_in_hand_image": [ 64, 64, 3 ], "robot0_gripper_qpos": [ 2 ], "state": [ 9 ] }, "offline_evaldir": "", "offline_traindir": "", "opt": "adam", "opt_eps": 1e-08, "outFolder": "logs/experiments", "parallel": false, "plotFigure": null, "precision": 32, "prefill": 5000, "pretrain": 100, "pretrain_actor_steps": 0, "pretrain_annealing": null, "pretrain_bc_loss_only": true, "pretrain_ema": false, "pretrain_joint_steps": 400000, "pretrain_loss": "ce", "pretrain_on_random": false, "pretrain_on_random_mixed": false, "pretrain_reward_val": null, "pretrain_separately": false, "priv_dim": 3, "randomSeed": 0, "recon_pretrain": true, "rep_scale": 0.1, "reset_every": 0, "residual_discount": 0.95, "residual_init_zeros": false, "reward_EMA": true, "reward_ensemble_size": 1, "reward_ensemble_subsample": 1, "reward_head": { "dist": "symlog_disc", "layers": 2, "loss_scale": 1.0, "outscale": 0.0 }, "root_dir": "/data/robocasa", "seed": 0, "seed_in_batches": true, "separate_reward_training": true, "shape_rewards": true, "shift_rewards": false, "showTime": null, "size": [ 128, 128 ], "speed": 1, "state_keys": [ "robot0_eef_pos", "robot0_eef_quat", "robot0_gripper_qpos" ], "steps": 1000000.0, "steps_per_batch": 100, "storeFigure": true, "success_data": "demo_im128_success_im128_front_wrist_view.hdf5", "targetRadius": 0.5, "task": "PnPCounterToSink_v0", "terminalType": "g", "time_limit": 100, "train_ratio": 1024, "train_residuals": true, "traindir": null, "transformer": { "activation": "gelu", "attn_dropout": 0.05, "block_output_dropout": 0.05, "causal": true, "context_length": 1500, "emb_dropout": 0.05, "embed_dim": 64, "enabled": true, "nn_parameter_for_timesteps": false, "num_heads": 1, "num_layers": 1, "pred_future_acs": false, "sinusoidal_embedding": true, "supervise_all_steps": false }, "turnRadius": 0.8, "u_max": 1.25, "unimix_ratio": 0.01, "units": 512, "updateTimes": 20, "use_reward": false, "use_additional_projection": false, "utd_ratio": 1, "validation_mse_trajs": 27, "video_pred_log": true, "visible_arm": false, "warmup": true, "warmupIter": 500, "weight_decay": 0.0, "wm": true, "x_max": 1.1, "x_min": -1.1, "y_max": 1.1, "y_min": -1.1 } } }