| recon: |
| spec: |
| _target_: far.builders.data.build_dataspec |
| local_pos_stats: |
| min: |
| - -2.5 |
| - -4 |
| max: |
| - 5 |
| - 4 |
| meters_per_waypoint: 0.25 |
| max_frame_offset: 128 |
| len_traj_pred: ${len_traj_pred} |
| planner_mu_init: |
| - -0.1 |
| - 0 |
| - 0 |
| planner_sigma_init: |
| - 0.02 |
| - 0.1 |
| - 0.3142 |
| pose_fields: |
| - x |
| - 'y' |
| - z |
| - pitch |
| - yaw |
| pose_dim: 5 |
| pos_dim: 2 |
| angle_dim: 1 |
| pose_mode: 2d |
| fps: 4.0 |
| shared: |
| manifest_path: results/manifests/recon/recon_split.json |
| backend: |
| _target_: far.data.backends.NavigationBackend |
| transform: |
| _target_: far.data.transforms.make_centercrop_resize |
| image_height: ${image_height} |
| image_width: ${image_width} |
| sampler: |
| context_goal: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 12 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: ${goals_per_obs} |
| context_traj: |
| _target_: far.data.samplers.ContextTrajectorySampler |
| context_pool_size: 12 |
| len_traj_pred: ${len_traj_pred} |
| formatter: |
| context_pose: |
| _target_: far.data.formatters.ContextPoseFormatter2D |
| spec: ${recon.spec} |
| train: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${recon.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${recon.shared.manifest_path} |
| index_path: results/indices/recon/recon_train.json |
| backend: ${recon.shared.backend} |
| sampler: ${recon.sampler.context_goal} |
| transform: ${recon.shared.transform} |
| test: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${recon.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${recon.shared.manifest_path} |
| index_path: results/indices/recon/recon_test.json |
| backend: ${recon.shared.backend} |
| sampler: ${recon.sampler.context_goal} |
| transform: ${recon.shared.transform} |
| eval: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${recon.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${recon.shared.manifest_path} |
| index_path: results/indices/recon/recon_test_time.json |
| backend: ${recon.shared.backend} |
| sampler: ${recon.sampler.context_traj} |
| transform: ${recon.shared.transform} |
| eval_traj: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${recon.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${recon.shared.manifest_path} |
| index_path: results/indices/recon/recon_test_plan.json |
| backend: ${recon.shared.backend} |
| sampler: ${recon.sampler.context_traj} |
| transform: ${recon.shared.transform} |
| loopnav: |
| spec: |
| _target_: far.builders.data.build_dataspec |
| local_pos_stats: |
| min: |
| - -2.5 |
| - -4 |
| - 0 |
| max: |
| - 5 |
| - 4 |
| - 1 |
| meters_per_waypoint: 0.25 |
| max_frame_offset: 128 |
| len_traj_pred: ${len_traj_pred} |
| planner_mu_init: |
| - -0.1 |
| - 0 |
| - 0 |
| - 0 |
| - 0 |
| planner_sigma_init: |
| - 0.02 |
| - 0.1 |
| - 0 |
| - 0 |
| - 0.3142 |
| pose_fields: |
| - x |
| - 'y' |
| - z |
| - pitch |
| - yaw |
| pose_dim: 5 |
| pos_dim: 3 |
| angle_dim: 2 |
| pose_mode: 3d |
| fps: 20.0 |
| shared: |
| manifest_path: results/manifests/loopnav/loopnav_split.json |
| backend: |
| _target_: far.data.backends.LoopNavBackend |
| standard_frame: false |
| transform: |
| _target_: far.data.transforms.make_resize |
| image_height: ${image_height} |
| image_width: ${image_width} |
| sampler: |
| random_clip: |
| _target_: far.data.samplers.RandomClipSampler |
| clip_len: 16 |
| frame_stride: 10 |
| context_goal: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 100 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: ${goals_per_obs} |
| context_traj: |
| _target_: far.data.samplers.ContextTrajectorySampler |
| context_pool_size: 100 |
| len_traj_pred: ${len_traj_pred} |
| formatter: |
| frame: |
| _target_: far.data.formatters.FrameFormatter |
| context_pose: |
| _target_: far.data.formatters.ContextPoseFormatter |
| spec: ${loopnav.spec} |
| train_tokenizer: |
| _target_: far.data.datasets.FormattedIterableDataset |
| formatter: ${loopnav.formatter.frame} |
| base_dataset: |
| _target_: far.data.datasets.WebVideoIterableDataset |
| manifest_path: results/manifests/loopnav/loopnav.json |
| index_path: results/indices/loopnav/loopnav_train_ABCA.json |
| backend: |
| _target_: far.data.backends.WebDatasetBackend |
| sampler: ${loopnav.sampler.random_clip} |
| transform: ${loopnav.shared.transform} |
| cache_dir: null |
| cache_size: 100 |
| shuffle_shards: 32 |
| shuffle_samples: 100 |
| loopnav_latent: |
| spec: |
| _target_: far.builders.data.build_dataspec |
| local_pos_stats: |
| min: |
| - 0 |
| - 0 |
| - 0 |
| max: |
| - 1 |
| - 1 |
| - 1 |
| meters_per_waypoint: 1.0 |
| max_frame_offset: 20.0 |
| len_traj_pred: ${len_traj_pred} |
| planner_mu_init: |
| - -0.1 |
| - 0 |
| - 0 |
| - 0 |
| - 0 |
| planner_sigma_init: |
| - 0.02 |
| - 0.1 |
| - 0 |
| - 0 |
| - 0.3142 |
| pose_fields: |
| - x |
| - 'y' |
| - z |
| - pitch |
| - yaw |
| pose_dim: 5 |
| pos_dim: 3 |
| angle_dim: 2 |
| pose_mode: 3d |
| fps: 20.0 |
| shared: |
| manifest_path: results/manifests/loopnav/loopnav_latent.json |
| backend: |
| _target_: far.data.backends.LatentLoopNavBackend |
| key_suffix: null |
| standard_frame: false |
| transform: null |
| sampler: |
| random_clip: |
| _target_: far.data.samplers.RandomClipSampler |
| clip_len: 8 |
| frame_stride: 10 |
| context_goal: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 200 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: ${goals_per_obs} |
| context_goal_test: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 200 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: 4 |
| context_traj: |
| _target_: far.data.samplers.ContextTrajectorySampler |
| context_pool_size: 200 |
| len_traj_pred: ${len_traj_pred} |
| formatter: |
| frame: |
| _target_: far.data.formatters.FrameFormatter |
| context_pose: |
| _target_: far.data.formatters.ContextPoseFormatter |
| spec: ${loopnav_latent.spec} |
| train: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${loopnav_latent.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${loopnav_latent.shared.manifest_path} |
| index_path: results/indices/loopnav/loopnav_latent_train.json |
| backend: ${loopnav_latent.shared.backend} |
| sampler: ${loopnav_latent.sampler.context_goal} |
| transform: ${loopnav_latent.shared.transform} |
| test: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${loopnav_latent.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${loopnav_latent.shared.manifest_path} |
| index_path: results/indices/loopnav/loopnav_latent_test.json |
| backend: ${loopnav_latent.shared.backend} |
| sampler: ${loopnav_latent.sampler.context_goal_test} |
| transform: ${loopnav_latent.shared.transform} |
| soundspaces_latent: |
| spec: |
| _target_: far.builders.data.build_dataspec |
| local_pos_stats: |
| min: |
| - 0 |
| - 0 |
| - 0 |
| max: |
| - 1 |
| - 1 |
| - 1 |
| meters_per_waypoint: 1.0 |
| max_frame_offset: 10.0 |
| len_traj_pred: ${len_traj_pred} |
| planner_mu_init: |
| - -0.1 |
| - 0 |
| - 0 |
| - 0 |
| - 0 |
| planner_sigma_init: |
| - 0.02 |
| - 0.1 |
| - 0 |
| - 0 |
| - 0.3142 |
| pose_fields: |
| - x |
| - 'y' |
| - z |
| - pitch |
| - yaw |
| pose_dim: 5 |
| pos_dim: 3 |
| angle_dim: 2 |
| pose_mode: 3d |
| fps: 10.0 |
| shared: |
| manifest_path: results/manifests/soundspaces/soundspaces_latent.json |
| backend: |
| _target_: far.data.backends.SoundSpacesBackend |
| key_suffix: null |
| audio_mode: null |
| audio_window_hops: 48 |
| transform: null |
| sampler: |
| random_clip: |
| _target_: far.data.samplers.RandomClipSampler |
| clip_len: 16 |
| frame_stride: 5 |
| context_goal: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 200 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: ${goals_per_obs} |
| context_goal_test: |
| _target_: far.data.samplers.ContextGoalSampler |
| context_pool_size: 200 |
| len_traj_pred: ${len_traj_pred} |
| goals_per_obs: 4 |
| context_traj: |
| _target_: far.data.samplers.ContextTrajectorySampler |
| context_pool_size: 200 |
| len_traj_pred: ${len_traj_pred} |
| formatter: |
| frame: |
| _target_: far.data.formatters.FrameFormatter |
| context_pose: |
| _target_: far.data.formatters.ContextPoseFormatter |
| spec: ${soundspaces_latent.spec} |
| train: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${soundspaces_latent.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${soundspaces_latent.shared.manifest_path} |
| index_path: results/indices/soundspaces/soundspaces_latent_train.json |
| backend: ${soundspaces_latent.shared.backend} |
| sampler: ${soundspaces_latent.sampler.context_goal} |
| transform: ${soundspaces_latent.shared.transform} |
| test: |
| _target_: far.data.datasets.FormattedVideoDataset |
| formatter: ${soundspaces_latent.formatter.context_pose} |
| base_dataset: |
| _target_: far.data.datasets.VideoDataset |
| manifest_path: ${soundspaces_latent.shared.manifest_path} |
| index_path: results/indices/soundspaces/soundspaces_latent_test.json |
| backend: ${soundspaces_latent.shared.backend} |
| sampler: ${soundspaces_latent.sampler.context_goal_test} |
| transform: ${soundspaces_latent.shared.transform} |
| model: |
| memory: |
| _target_: far.memories.pnp_cl.PnPRetrieval |
| ckpt_path: null |
| chunk_size: 20 |
| num_sets: 4 |
| softmax_temperature: 1.0 |
| scoring: visual |
| metadata_hidden: 128 |
| local_pos_stats: null |
| stride_to_seconds: 1.0 |
| encoder_mode: frozen |
| adapter_hidden: 384 |
| query_source: frame |
| gate_eps: 0.05 |
| query_vit: |
| _target_: far.memories.pnp_cl.QueryViT |
| img_height: 18 |
| img_width: 32 |
| patch_size: 2 |
| in_chans: 16 |
| embed_dim: 384 |
| depth: 6 |
| num_heads: 12 |
| action_dim: ${action_dim} |
| key_dim: 256 |
| mlp_ratio: 4.0 |
| wandb: |
| project: PMem-World |
| name: null |
| entity: null |
| mode: online |
| group: loopnav_latent |
| tags: |
| - loopnav_latent |
| - retriever_pretrain |
| - longlive |
| run_name: retriever_longlive_pretrain |
| context_size: 4 |
| action_dim: 5 |
| pose_dim: 3 |
| image_height: 224 |
| image_width: 224 |
| len_traj_pred: 128 |
| goals_per_obs: 4 |
| lr: 0.0003 |
| weight_decay: 0.0001 |
| grad_clip_val: 1.0 |
| iterations: 400000 |
| warmup_steps: 0 |
| from_checkpoint: null |
| global_seed: 0 |
| log_every: 100 |
| ckpt_every: 25000 |
| eval_every: 5000 |
| bfloat16: false |
| batch_size: 32 |
| num_workers: 12 |
| datasets: |
| - loopnav_latent |
| contrastive: |
| chunk_size: ${model.memory.chunk_size} |
| positive_pair_type: |
| - temporal |
| - pose |
| - jepa |
| num_positives: 4 |
| num_negatives: 16 |
| cl_temperature: 0.1 |
| cl_pose_radius: 30.0 |
| cl_pose_fov_half_h: 52.5 |
| cl_pose_fov_half_v: 37.5 |
| cl_pose_num_samples: 1024 |
| local_pos_stats: null |
| longlive: |
| hidden_dims: |
| - 64 |
| - 128 |
| - 256 |
| delta_weight: 1.0 |
| delta_margin: 0.85 |
| delta_window: 3 |
| smooth_weight: 1.0 |
|
|
