diff --git "a/train.log" "b/train.log" new file mode 100644--- /dev/null +++ "b/train.log" @@ -0,0 +1,11637 @@ +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:03:13 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:03:13 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:03:13 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:03:13 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:03:13 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:03:13 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:03:13 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:03:14 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:03:14 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:03:14 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:03:14 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:03:14 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:03:15 - ERROR - stderr - Loading checkpoint shards: 0%| | 0/2 [00:00', 'DEFAULT_POINT_START_TOKEN': '', 'DEFAULT_POINT_END_TOKEN': '', 'output_dir': 'epoch6/PointLLM_train_stage1/PointLLM_train_stagece', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': 'None', 'eval_delay': 0, 'learning_rate': 0.001, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 6.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'info', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'epoch6/PointLLM_train_stage1/PointLLM_train_stagece/runs/Oct06_15-02-09_6e49fa911257', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'no', 'save_steps': 400, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'use_mps_device': False, 'seed': 42, 'data_seed': 'None', 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'cuda_amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': 0, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 'None', 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'PointLLM_train_stagece', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'fsdp': '[]', 'fsdp_min_num_params': 0, 'fsdp_config': "{'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}", 'fsdp_transformer_layer_cls_to_wrap': 'None', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': 'None', 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': "['wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'gradient_checkpointing': True, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': 'None', 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': 'None', 'torch_compile_mode': 'None', 'cache_dir': 'None', 'model_max_length': 2048, 'model_debug': False, 'fix_llm': True, 'fix_pointnet': True, 'force_fsdp': False, 'tune_mm_mlp_adapter': True, 'stage_2': False, 'pretrained_mm_mlp_adapter': 'None', 'detatch_point_token': '', 'train_batch_size': 16, 'eval_batch_size': 4} +2025-10-06 15:04:37 - ERROR - stderr - 0%| | 0/2088 [00:00 +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train_mem.py", line 13, in +2025-10-06 15:05:44 - ERROR - stderr - Original exception was: +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train_mem.py", line 13, in +2025-10-06 15:05:44 - ERROR - stderr - Original exception was: +2025-10-06 15:05:44 - ERROR - stderr - Traceback (most recent call last): +2025-10-06 15:05:44 - ERROR - stderr - train() +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - ERROR - stderr - Traceback (most recent call last): +2025-10-06 15:05:44 - ERROR - stderr - train() +2025-10-06 15:05:44 - ERROR - stderr - train() +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train.py", line 210, in train +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train.py", line 210, in train +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train_mem.py", line 13, in +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train_mem.py", line 13, in +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train.py", line 210, in train +2025-10-06 15:05:44 - ERROR - stderr - +2025-10-06 15:05:44 - ERROR - stderr - trainer.train() +2025-10-06 15:05:44 - ERROR - stderr - train() +2025-10-06 15:05:44 - ERROR - stderr - trainer.train() +2025-10-06 15:05:44 - ERROR - stderr - trainer.train() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1644, in train +2025-10-06 15:05:44 - ERROR - stderr - Original exception was: +2025-10-06 15:05:44 - ERROR - stderr - train() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1644, in train +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train.py", line 210, in train +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1644, in train +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train.py", line 210, in train +2025-10-06 15:05:44 - ERROR - stderr - Traceback (most recent call last): +2025-10-06 15:05:44 - ERROR - stderr - trainer.train() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1644, in train +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train_mem.py", line 13, in +2025-10-06 15:05:44 - ERROR - stderr - trainer.train() +2025-10-06 15:05:44 - ERROR - stderr - return inner_training_loop( +2025-10-06 15:05:44 - ERROR - stderr - return inner_training_loop( +2025-10-06 15:05:44 - ERROR - stderr - return inner_training_loop( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1881, in _inner_training_loop +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1644, in train +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1881, in _inner_training_loop +2025-10-06 15:05:44 - ERROR - stderr - train() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1881, in _inner_training_loop +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train.py", line 210, in train +2025-10-06 15:05:44 - ERROR - stderr - return inner_training_loop( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1881, in _inner_training_loop +2025-10-06 15:05:44 - ERROR - stderr - for step, inputs in enumerate(epoch_iterator): +2025-10-06 15:05:44 - ERROR - stderr - trainer.train() +2025-10-06 15:05:44 - ERROR - stderr - for step, inputs in enumerate(epoch_iterator): +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ +2025-10-06 15:05:44 - ERROR - stderr - for step, inputs in enumerate(epoch_iterator): +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1644, in train +2025-10-06 15:05:44 - ERROR - stderr - return inner_training_loop( +2025-10-06 15:05:44 - ERROR - stderr - data = self._next_data() +2025-10-06 15:05:44 - ERROR - stderr - for step, inputs in enumerate(epoch_iterator): +2025-10-06 15:05:44 - ERROR - stderr - data = self._next_data() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data +2025-10-06 15:05:44 - ERROR - stderr - data = self._next_data() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1881, in _inner_training_loop +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data +2025-10-06 15:05:44 - ERROR - stderr - data = self._dataset_fetcher.fetch(index) # may raise StopIteration +2025-10-06 15:05:44 - ERROR - stderr - data = self._next_data() +2025-10-06 15:05:44 - ERROR - stderr - data = self._dataset_fetcher.fetch(index) # may raise StopIteration +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch +2025-10-06 15:05:44 - ERROR - stderr - data = self._dataset_fetcher.fetch(index) # may raise StopIteration +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in +2025-10-06 15:05:44 - ERROR - stderr - data = self._dataset_fetcher.fetch(index) # may raise StopIteration +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - return inner_training_loop( +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 236, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 236, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - for step, inputs in enumerate(epoch_iterator): +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 236, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1881, in _inner_training_loop +2025-10-06 15:05:44 - ERROR - stderr - data_dict = preprocess_v1( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ +2025-10-06 15:05:44 - ERROR - stderr - data_dict = preprocess_v1( +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/utils.py", line 109, in preprocess_v1 +2025-10-06 15:05:44 - ERROR - stderr - data_dict = preprocess_v1( +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/utils.py", line 109, in preprocess_v1 +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 220, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/utils.py", line 108, in preprocess_v1 +2025-10-06 15:05:44 - ERROR - stderr - instruction_len = len(tokenizer(parts[0]).input_ids) - 2 +2025-10-06 15:05:44 - ERROR - stderr - instruction_len = len(tokenizer(parts[0]).input_ids) - 2 +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2531, in __call__ +2025-10-06 15:05:44 - ERROR - stderr - round_len = len(tokenizer(rou).input_ids) +2025-10-06 15:05:44 - ERROR - stderr - point_cloud = self._load_point_cloud(object_id) # * N, C +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2531, in __call__ +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2531, in __call__ +2025-10-06 15:05:44 - ERROR - stderr - data = self._next_data() +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 149, in _load_point_cloud +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data +2025-10-06 15:05:44 - ERROR - stderr - return self._load_objaverse_point_cloud(object_id) +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 155, in _load_objaverse_point_cloud +2025-10-06 15:05:44 - ERROR - stderr - point_cloud = npz_file['points'] +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/npyio.py", line 256, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - for step, inputs in enumerate(epoch_iterator): +2025-10-06 15:05:44 - ERROR - stderr - data = self._dataset_fetcher.fetch(index) # may raise StopIteration +2025-10-06 15:05:44 - ERROR - stderr - encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs) +2025-10-06 15:05:44 - ERROR - stderr - encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs) +2025-10-06 15:05:44 - ERROR - stderr - encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2637, in _call_one +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2637, in _call_one +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2637, in _call_one +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in +2025-10-06 15:05:44 - ERROR - stderr - data = self._next_data() +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - return self.encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - return self.encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2710, in encode_plus +2025-10-06 15:05:44 - ERROR - stderr - return self.encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 236, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2710, in encode_plus +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2710, in encode_plus +2025-10-06 15:05:44 - ERROR - stderr - data_dict = preprocess_v1( +2025-10-06 15:05:44 - ERROR - stderr - data = self._dataset_fetcher.fetch(index) # may raise StopIteration +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/utils.py", line 81, in preprocess_v1 +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch +2025-10-06 15:05:44 - ERROR - stderr - return self._encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - return self._encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 649, in _encode_plus +2025-10-06 15:05:44 - ERROR - stderr - return self._encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - input_ids = tokenizer( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 649, in _encode_plus +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 649, in _encode_plus +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2531, in __call__ +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - first_ids = get_input_ids(text) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in +2025-10-06 15:05:44 - ERROR - stderr - first_ids = get_input_ids(text) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 616, in get_input_ids +2025-10-06 15:05:44 - ERROR - stderr - first_ids = get_input_ids(text) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 616, in get_input_ids +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 616, in get_input_ids +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 236, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokenize(text, **kwargs) +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokenize(text, **kwargs) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 517, in tokenize +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 517, in tokenize +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokens_trie.split(text) +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokenize(text, **kwargs) +2025-10-06 15:05:44 - ERROR - stderr - data_dict = preprocess_v1( +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokens_trie.split(text) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 182, in split +2025-10-06 15:05:44 - ERROR - stderr - encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 152, in split +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/utils.py", line 109, in preprocess_v1 +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 517, in tokenize +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2617, in _call_one +2025-10-06 15:05:44 - ERROR - stderr - while next_char in looktrie_pointer: +2025-10-06 15:05:44 - ERROR - stderr - if "" in trie_pointer: +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - ERROR - stderr - instruction_len = len(tokenizer(parts[0]).input_ids) - 2 +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2531, in __call__ +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokens_trie.split(text) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 152, in split +2025-10-06 15:05:44 - ERROR - stderr - return self.batch_encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2808, in batch_encode_plus +2025-10-06 15:05:44 - ERROR - stderr - if "" in trie_pointer: +2025-10-06 15:05:44 - ERROR - stderr - encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs) +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2637, in _call_one +2025-10-06 15:05:44 - ERROR - stderr - return self.encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2710, in encode_plus +2025-10-06 15:05:44 - ERROR - stderr - return self._encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 649, in _encode_plus +2025-10-06 15:05:44 - ERROR - stderr - first_ids = get_input_ids(text) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 616, in get_input_ids +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokenize(text, **kwargs) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 517, in tokenize +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokens_trie.split(text) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line -1, in split +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - ERROR - stderr - return format.read_array(bytes, +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/format.py", line 831, in read_array +2025-10-06 15:05:44 - ERROR - stderr - return self._batch_encode_plus( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 733, in _batch_encode_plus +2025-10-06 15:05:44 - ERROR - stderr - first_ids = get_input_ids(ids) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 700, in get_input_ids +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokenize(text, **kwargs) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 517, in tokenize +2025-10-06 15:05:44 - ERROR - stderr - tokens = self.tokens_trie.split(text) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/tokenization_utils.py", line 208, in split +2025-10-06 15:05:44 - ERROR - stderr - states[start] = trie_pointer +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - ERROR - stderr - Traceback (most recent call last): +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train_mem.py", line 13, in +2025-10-06 15:05:44 - ERROR - stderr - train() +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train.py", line 210, in train +2025-10-06 15:05:44 - ERROR - stderr - trainer.train() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1644, in train +2025-10-06 15:05:44 - ERROR - stderr - return inner_training_loop( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1881, in _inner_training_loop +2025-10-06 15:05:44 - ERROR - stderr - for step, inputs in enumerate(epoch_iterator): +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ +2025-10-06 15:05:44 - ERROR - stderr - data = self._next_data() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data +2025-10-06 15:05:44 - ERROR - stderr - data = self._dataset_fetcher.fetch(index) # may raise StopIteration +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 220, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - point_cloud = self._load_point_cloud(object_id) # * N, C +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 149, in _load_point_cloud +2025-10-06 15:05:44 - ERROR - stderr - return self._load_objaverse_point_cloud(object_id) +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 154, in _load_objaverse_point_cloud +2025-10-06 15:05:44 - ERROR - stderr - npz_file = np.load(os.path.join(self.data_path, filename)) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/npyio.py", line 434, in load +2025-10-06 15:05:44 - ERROR - stderr - magic = fid.read(N) +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - ERROR - stderr - data = _read_bytes(fp, read_size, "array data") +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/format.py", line 966, in _read_bytes +2025-10-06 15:05:44 - ERROR - stderr - r = fp.read(size - len(data)) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/zipfile.py", line 930, in read +2025-10-06 15:05:44 - ERROR - stderr - data = self._read1(n) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/zipfile.py", line 1006, in _read1 +2025-10-06 15:05:44 - ERROR - stderr - data = self._decompressor.decompress(data, n) +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - ERROR - stderr - Traceback (most recent call last): +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train_mem.py", line 13, in +2025-10-06 15:05:44 - ERROR - stderr - train() +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train.py", line 210, in train +2025-10-06 15:05:44 - ERROR - stderr - trainer.train() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1644, in train +2025-10-06 15:05:44 - ERROR - stderr - return inner_training_loop( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1881, in _inner_training_loop +2025-10-06 15:05:44 - ERROR - stderr - for step, inputs in enumerate(epoch_iterator): +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ +2025-10-06 15:05:44 - ERROR - stderr - data = self._next_data() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data +2025-10-06 15:05:44 - ERROR - stderr - data = self._dataset_fetcher.fetch(index) # may raise StopIteration +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 220, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - point_cloud = self._load_point_cloud(object_id) # * N, C +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 149, in _load_point_cloud +2025-10-06 15:05:44 - ERROR - stderr - return self._load_objaverse_point_cloud(object_id) +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 155, in _load_objaverse_point_cloud +2025-10-06 15:05:44 - ERROR - stderr - point_cloud = npz_file['points'] +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/npyio.py", line 256, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - return format.read_array(bytes, +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/format.py", line 831, in read_array +2025-10-06 15:05:44 - ERROR - stderr - data = _read_bytes(fp, read_size, "array data") +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/format.py", line 966, in _read_bytes +2025-10-06 15:05:44 - ERROR - stderr - r = fp.read(size - len(data)) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/zipfile.py", line 930, in read +2025-10-06 15:05:44 - ERROR - stderr - data = self._read1(n) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/zipfile.py", line 1006, in _read1 +2025-10-06 15:05:44 - ERROR - stderr - data = self._decompressor.decompress(data, n) +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - ERROR - stderr - Traceback (most recent call last): +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train_mem.py", line 13, in +2025-10-06 15:05:44 - ERROR - stderr - train() +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/train/train.py", line 210, in train +2025-10-06 15:05:44 - ERROR - stderr - trainer.train() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1644, in train +2025-10-06 15:05:44 - ERROR - stderr - return inner_training_loop( +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/transformers/trainer.py", line 1881, in _inner_training_loop +2025-10-06 15:05:44 - ERROR - stderr - for step, inputs in enumerate(epoch_iterator): +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ +2025-10-06 15:05:44 - ERROR - stderr - data = self._next_data() +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data +2025-10-06 15:05:44 - ERROR - stderr - data = self._dataset_fetcher.fetch(index) # may raise StopIteration +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in fetch +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 51, in +2025-10-06 15:05:44 - ERROR - stderr - data = [self.dataset[idx] for idx in possibly_batched_index] +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 220, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - point_cloud = self._load_point_cloud(object_id) # * N, C +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 149, in _load_point_cloud +2025-10-06 15:05:44 - ERROR - stderr - return self._load_objaverse_point_cloud(object_id) +2025-10-06 15:05:44 - ERROR - stderr - File "/data1/xindanzhang/PointLLM/pointllm4d/data/object_point_dataset.py", line 155, in _load_objaverse_point_cloud +2025-10-06 15:05:44 - ERROR - stderr - point_cloud = npz_file['points'] +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/npyio.py", line 256, in __getitem__ +2025-10-06 15:05:44 - ERROR - stderr - return format.read_array(bytes, +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/format.py", line 831, in read_array +2025-10-06 15:05:44 - ERROR - stderr - data = _read_bytes(fp, read_size, "array data") +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/site-packages/numpy/lib/format.py", line 966, in _read_bytes +2025-10-06 15:05:44 - ERROR - stderr - r = fp.read(size - len(data)) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/zipfile.py", line 930, in read +2025-10-06 15:05:44 - ERROR - stderr - data = self._read1(n) +2025-10-06 15:05:44 - ERROR - stderr - File "/home/xindanzhang/anaconda3/envs/pointllm/lib/python3.10/zipfile.py", line 1006, in _read1 +2025-10-06 15:05:44 - ERROR - stderr - data = self._decompressor.decompress(data, n) +2025-10-06 15:05:44 - ERROR - stderr - KeyboardInterrupt +2025-10-06 15:05:44 - INFO - wandb.sdk.lib.service.service_client - Reached EOF. +2025-10-06 15:05:44 - INFO - wandb.sdk.mailbox.mailbox - Closing mailbox, abandoning 1 handles. +2025-10-06 15:05:44 - INFO - wandb.sdk.mailbox.mailbox - Closing mailbox, abandoning 1 handles. +2025-10-06 15:07:46 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:07:46 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:07:46 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:07:46 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:07:46 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:07:46 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:07:47 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:07:47 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:07:47 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:07:47 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:07:47 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:07:48 - INFO - pointllm4d.model.pointllm - Using PointBERT. +2025-10-06 15:07:48 - INFO - stdout - Loading PointBERT config from /data1/xindanzhang/PointLLM/pointllm4d/model/pointbert/PointTransformer_8192point_2layer.yaml. +2025-10-06 15:07:48 - INFO - pointllm4d.model.pointllm - Using 6 dim of points. +2025-10-06 15:07:48 - INFO - pointllm4d.model.pointllm - Use max pool is False. Number of point token is 513. +2025-10-06 15:07:48 - INFO - pointllm4d.model.pointllm - Point backbone output dim: 384. +2025-10-06 15:07:48 - INFO - pointllm4d.model.pointllm - Point projector output dim: 4096. +2025-10-06 15:07:48 - ERROR - stderr - Loading checkpoint shards: 0%| | 0/2 [00:00', 'DEFAULT_POINT_START_TOKEN': '', 'DEFAULT_POINT_END_TOKEN': '', 'output_dir': 'epoch6/PointLLM_train_stage1/PointLLM_train_stagece', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': 'None', 'eval_delay': 0, 'learning_rate': 0.001, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 6.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'info', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'epoch6/PointLLM_train_stage1/PointLLM_train_stagece/runs/Oct06_15-06-42_6e49fa911257', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 400, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'use_mps_device': False, 'seed': 42, 'data_seed': 'None', 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'cuda_amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': 0, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 'None', 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'PointLLM_train_stagece', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'fsdp': '[]', 'fsdp_min_num_params': 0, 'fsdp_config': "{'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}", 'fsdp_transformer_layer_cls_to_wrap': 'None', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': 'None', 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': "['wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': False, 'gradient_checkpointing': True, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': 'None', 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': 'None', 'torch_compile_mode': 'None', 'cache_dir': 'None', 'model_max_length': 2048, 'model_debug': False, 'fix_llm': True, 'fix_pointnet': True, 'force_fsdp': False, 'tune_mm_mlp_adapter': True, 'stage_2': False, 'pretrained_mm_mlp_adapter': 'None', 'detatch_point_token': '', 'train_batch_size': 16, 'eval_batch_size': 4} +2025-10-06 15:08:31 - ERROR - stderr - 0%| | 0/2088 [00:00