InstaDeepAI

/

BulkRNABert

like 2

Follow

InstaDeep Ltd 366

Hello authors,

Thank you for this great model and the amazing work.
I am facing an error loading the model. Could you please help in resolving this? Thanks!
Script:

# Load model and tokenizer.
config = AutoConfig.from_pretrained(
    "InstaDeepAI/BulkRNABert",
    trust_remote_code=True,
)
config.embeddings_layers_to_save = (4,) # last transformer layer

tokenizer = AutoTokenizer.from_pretrained("InstaDeepAI/BulkRNABert", trust_remote_code=True)
model = AutoModel.from_pretrained(
    "InstaDeepAI/BulkRNABert",
    config=config,
    trust_remote_code=True,
)

Output :

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[6], line 9
      6 config.embeddings_layers_to_save = (4,) # last transformer layer
      8 tokenizer = AutoTokenizer.from_pretrained("InstaDeepAI/BulkRNABert", trust_remote_code=True)
----> 9 model = AutoModel.from_pretrained(
     10     "InstaDeepAI/BulkRNABert",
     11     config=config,
     12     trust_remote_code=True,
     13 )

File ~/miniconda3/envs/bulkRNAbert/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:366, in _BaseAutoModelClass.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    364         model_class.register_for_auto_class(auto_class=cls)
    365     model_class = add_generation_mixin_to_remote_model(model_class)
--> 366     return model_class.from_pretrained(
    367         pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
    368     )
    369 elif type(config) in cls._model_mapping:
    370     model_class = _get_model_class(config, cls._model_mapping)

File ~/miniconda3/envs/bulkRNAbert/lib/python3.10/site-packages/transformers/modeling_utils.py:248, in restore_default_dtype.<locals>._wrapper(*args, **kwargs)
    246 old_dtype = torch.get_default_dtype()
    247 try:
--> 248     return func(*args, **kwargs)
    249 finally:
    250     torch.set_default_dtype(old_dtype)

File ~/miniconda3/envs/bulkRNAbert/lib/python3.10/site-packages/transformers/modeling_utils.py:3988, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)
   3985     torch.set_default_dtype(dtype_orig)
   3987 # Finalize model weight initialization
-> 3988 model, missing_keys, unexpected_keys, mismatched_keys, offload_index, error_msgs = cls._load_pretrained_model(
   3989     model,
   3990     state_dict,
   3991     checkpoint_files,
   3992     pretrained_model_name_or_path,
   3993     ignore_mismatched_sizes=ignore_mismatched_sizes,
   3994     sharded_metadata=sharded_metadata,
   3995     device_map=device_map,
   3996     disk_offload_folder=offload_folder,
   3997     dtype=dtype,
   3998     hf_quantizer=hf_quantizer,
   3999     device_mesh=device_mesh,
   4000     weights_only=weights_only,
   4001     weight_mapping=weight_conversions,
   4002 )
   4004 model.eval()  # Set model in evaluation mode to deactivate DropOut modules by default
   4005 model.set_use_kernels(use_kernels, kernel_config)

File ~/miniconda3/envs/bulkRNAbert/lib/python3.10/site-packages/transformers/modeling_utils.py:4148, in PreTrainedModel._load_pretrained_model(cls, model, state_dict, checkpoint_files, pretrained_model_name_or_path, ignore_mismatched_sizes, sharded_metadata, device_map, disk_offload_folder, dtype, hf_quantizer, device_mesh, weights_only, weight_mapping)
   4145         k.__exit__(None, None, None)
   4147 # Marks tied weights as `_is_hf_initialized` to avoid initializing them (it's very important for efficiency)
-> 4148 model.mark_tied_weights_as_initialized()
   4150 # Move missing (and potentially mismatched) keys back to cpu from meta device (because they won't be moved when
   4151 # loading the weights as they are not in the loaded state dict)
   4152 miss_and_mismatched = missing_keys | {k[0] for k in mismatched_keys}

File ~/miniconda3/envs/bulkRNAbert/lib/python3.10/site-packages/transformers/modeling_utils.py:4475, in PreTrainedModel.mark_tied_weights_as_initialized(self)
   4470 def mark_tied_weights_as_initialized(self):
   4471     """Adds the `_is_hf_initialized` flag on parameters that will be tied, in order to avoid initializing them
   4472     later as they will be tied (overwritten) anyway.
   4473     This is very important as most embeddings are tied, and they are huge params (vocabularies are often 256k), so
   4474     running inits on them is very costly."""
-> 4475     for tied_param in self.all_tied_weights_keys.keys():
   4476         param = self.get_parameter(tied_param)
   4477         param._is_hf_initialized = True

File ~/miniconda3/envs/bulkRNAbert/lib/python3.10/site-packages/torch/nn/modules/module.py:1964, in Module.__getattr__(self, name)
   1962     if name in modules:
   1963         return modules[name]
-> 1964 raise AttributeError(
   1965     f"'{type(self).__name__}' object has no attribute '{name}'"
   1966 )

AttributeError: 'BulkRNABert' object has no attribute 'all_tied_weights_keys'