diff --git "a/Baseline_gpu3_resume.log" "b/Baseline_gpu3_resume.log" new file mode 100644--- /dev/null +++ "b/Baseline_gpu3_resume.log" @@ -0,0 +1,237 @@ +INFO 2026-07-15 14:32:49 ot_train.py:272 {'batch_size': 64, + 'checkpoint_path': '/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_6D_baseline', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/Abs_6D/Baseline', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [10], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 500, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': '1jmjb68j'}} +INFO 2026-07-15 14:32:51 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-15 14:32:51 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/1jmjb68j +INFO 2026-07-15 14:32:51 ot_train.py:298 Creating dataset +INFO 2026-07-15 14:32:54 ot_train.py:332 Creating policy +INFO 2026-07-15 14:32:54 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/Abs_6D/Baseline/checkpoints/050000/pretrained_model +INFO 2026-07-15 14:32:54 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00