Update modeling_isoformer.py

modeling_isoformer.py

@@ -54,30 +54,8 @@ class Isoformer(PreTrainedModel):
         )
         self.config = config
 
-        # self.enformer_config = EnformerConfig(
-        #     dim=config.enformer_dim,
-        #     depth=config.enformer_depth,
-        #     heads=config.enformer_heads,
-        #     output_heads=dict(
-        #         human=1,
-        #         mouse=1 # TODO CHANGE
-        #     ),
-        #     target_length=config.enformer_target_length,  # 896,
-        #     attn_dim_key=config.enformer_attn_dim_key,
-        #     dropout_rate=0.4,
-        #     attn_dropout=0.05,
-        #     pos_dropout=0.01,
-        #     use_checkpointing=config.enformer_use_checkpointing,
-        #     use_convnext=config.enformer_use_convnext,
-        #     num_downsamples=config.enformer_num_downsamples,
-        #     # genetic sequence is downsampled 2 ** 7 == 128x in default Enformer - can be changed for higher resolution
-        #     dim_divisible_by=config.enformer_dim_divisible_by,
-        #     use_tf_gamma=False,
-        # )
-
-        self.esm_model = EsmForMaskedLM(self.esm_config) # protein encoder
-        self.nt_model = NTForMaskedLM(self.nt_config) # rna encoder
-        #self.enformer_model = Enformer(self.enformer_config) # dna encoder
+        self.esm_model = EsmForMaskedLM(self.esm_config) 
+        self.nt_model = NTForMaskedLM(self.nt_config) 
         self.enformer_model = Enformer.from_pretrained("EleutherAI/enformer-official-rough")
 
         self.cross_attention_layer_rna = MultiHeadAttention(