Fix path

modeling_InternLM.py

@@ -1,10 +1,7 @@
-# This script is based on https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py
-""" PyTorch LLaMA model."""
 import math
 from typing import List, Union
 from typing import Optional, Tuple
 
-# from apex.normalization.fused_layer_norm import MixedFusedRMSNorm as LlamaRMSNorm
 import rotary_emb
 import torch
 import torch.utils.checkpoint
@@ -16,18 +13,14 @@ from torch.nn import CrossEntropyLoss
 from transformers.activations import ACT2FN
 from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
 from transformers.modeling_utils import PreTrainedModel
-from transformers.models.llama.configuration_llama import LlamaConfig
-from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings
 from transformers.utils import logging
 
-from modeling_utils import LoRALinear
-
-# from flash_attn.modules.mha import FlashSelfAttention
+from .modeling_utils import LoRALinear
+from .configuration_InternLM_XComposer import InternLMXComposerConfig
 
 logger = logging.get_logger(__name__)
 
-_CONFIG_FOR_DOC = "LlamaConfig"
-""" PyTorch LLaMA model."""
+_CONFIG_FOR_DOC = "InternLMXComposerConfig"
 
 
 class ApplyRotaryEmbQKV_(torch.autograd.Function):
@@ -77,7 +70,7 @@ class ApplyRotaryEmbQKV_(torch.autograd.Function):
         return dqkv, None, None, None, None
 
 
-class ConvertedLlamaRotaryEmbedding(torch.nn.Module):
+class ConvertedInternLMRotaryEmbedding(torch.nn.Module):
     def __init__(self, dim: int, base=10000, scale_base=0, device=None):
         """ """
         super().__init__()
@@ -168,9 +161,9 @@ apply_rotary_emb_qkv_ = ApplyRotaryEmbQKV_.apply
 legacy_apply_rotary_embed_qkv = LegacyApplyRotaryEmbQKV_.apply
 
 
-class InternConvertedLlamaAttention(nn.Module):
+class InternConvertedInternLMAttention(nn.Module):
     """Multi-headed attention from 'Attention Is All You Need' paper"""
-    def __init__(self, config: LlamaConfig):
+    def __init__(self, config: InternLMXComposerConfig):
         super().__init__()
         self.config = config
         self.hidden_size = config.hidden_size
@@ -244,7 +237,7 @@ class InternConvertedLlamaAttention(nn.Module):
                     bias=config.kqvo_bias,
                 )
 
-        self.rotary_emb = ConvertedLlamaRotaryEmbedding(self.head_dim)
+        self.rotary_emb = ConvertedInternLMRotaryEmbedding(self.head_dim)
 
     def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
         return tensor.view(bsz, seq_len, self.num_heads,
@@ -460,10 +453,10 @@ def _expand_mask(mask: torch.Tensor,
                                      torch.finfo(dtype).min)
 
 
-class LlamaRMSNorm(nn.Module):
+class InternLMRMSNorm(nn.Module):
     def __init__(self, hidden_size, eps=1e-6):
         """
-        LlamaRMSNorm is equivalent to T5LayerNorm
+        InternLMRMSNorm is equivalent to T5LayerNorm
         """
         super().__init__()
         self.weight = nn.Parameter(torch.ones(hidden_size))
@@ -482,7 +475,7 @@ class LlamaRMSNorm(nn.Module):
         return self.weight * hidden_states
 
 
-class LlamaRotaryEmbedding(torch.nn.Module):
+class InternLMRotaryEmbedding(torch.nn.Module):
     def __init__(self,
                  dim,
                  max_position_embeddings=2048,
@@ -550,9 +543,9 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
     return q_embed, k_embed
 
 
-class LlamaMLP(nn.Module):
+class InternLMMLP(nn.Module):
     def __init__(self, hidden_size: int, intermediate_size: int,
-                 hidden_act: str, config: LlamaConfig):
+                 hidden_act: str, config: InternLMXComposerConfig):
         super().__init__()
         self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
         if config.lora_cfg is not None and 'ffn' in config.lora_cfg[
@@ -579,9 +572,9 @@ class LlamaMLP(nn.Module):
         return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
 
 
-class LlamaAttention(nn.Module):
+class InternLMAttention(nn.Module):
     """Multi-headed attention from 'Attention Is All You Need' paper"""
-    def __init__(self, config: LlamaConfig):
+    def __init__(self, config: InternLMXComposerConfig):
         super().__init__()
         self.config = config
         self.hidden_size = config.hidden_size
@@ -648,7 +641,7 @@ class LlamaAttention(nn.Module):
                                         self.hidden_size,
                                         bias=False)
 
-        self.rotary_emb = LlamaRotaryEmbedding(
+        self.rotary_emb = InternLMRotaryEmbedding(
             self.head_dim,
             max_position_embeddings=self.max_position_embeddings)
 
@@ -731,25 +724,25 @@ class LlamaAttention(nn.Module):
         return attn_output, attn_weights, past_key_value
 
 
-class LlamaDecoderLayer(nn.Module):
-    def __init__(self, config: LlamaConfig):
+class InternLMDecoderLayer(nn.Module):
+    def __init__(self, config: InternLMXComposerConfig):
         super().__init__()
         self.hidden_size = config.hidden_size
         if hasattr(config,
                    'intern_converted_llm') and config.intern_converted_llm:
-            self.self_attn = InternConvertedLlamaAttention(config=config)
+            self.self_attn = InternConvertedInternLMAttention(config=config)
         else:
-            self.self_attn = LlamaAttention(config=config)
-        self.mlp = LlamaMLP(
+            self.self_attn = InternLMAttention(config=config)
+        self.mlp = InternLMMLP(
             hidden_size=self.hidden_size,
             intermediate_size=config.intermediate_size,
             hidden_act=config.hidden_act,
             config=config,
         )
-        self.input_layernorm = LlamaRMSNorm(config.hidden_size,
-                                            eps=config.rms_norm_eps)
-        self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size,
-                                                     eps=config.rms_norm_eps)
+        self.input_layernorm = InternLMRMSNorm(config.hidden_size,
+                                               eps=config.rms_norm_eps)
+        self.post_attention_layernorm = InternLMRMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps)
 
     def forward(
         self,
@@ -807,32 +800,11 @@ class LlamaDecoderLayer(nn.Module):
         return outputs
 
 
-LLAMA_START_DOCSTRING = r"""
-    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
-    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
-    etc.)
-
-    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
-    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
-    and behavior.
-
-    Parameters:
-        config ([`LlamaConfig`]):
-            Model configuration class with all the parameters of the model. Initializing with a config file does not
-            load the weights associated with the model, only the configuration. Check out the
-            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
-"""
-
-
-@add_start_docstrings(
-    "The bare LLaMA Model outputting raw hidden-states without any specific head on top.",
-    LLAMA_START_DOCSTRING,
-)
-class LlamaPreTrainedModel(PreTrainedModel):
-    config_class = LlamaConfig
+class InternLMPreTrainedModel(PreTrainedModel):
+    config_class = InternLMXComposerConfig
     base_model_prefix = "model"
     supports_gradient_checkpointing = True
-    _no_split_modules = ["LlamaDecoderLayer"]
+    _no_split_modules = ["InternLMDecoderLayer"]
     _keys_to_ignore_on_load_unexpected = [r"decoder\.version"]
 
     def _init_weights(self, module):
@@ -847,86 +819,18 @@ class LlamaPreTrainedModel(PreTrainedModel):
                 module.weight.data[module.padding_idx].zero_()
 
     def _set_gradient_checkpointing(self, module, value=False):
-        if isinstance(module, LlamaModel):
+        if isinstance(module, InternLMModel):
             module.gradient_checkpointing = value
 
 
-LLAMA_INPUTS_DOCSTRING = r"""
-    Args:
-        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
-            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
-            it.
-
-            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
-            [`PreTrainedTokenizer.__call__`] for details.
-
-            [What are input IDs?](../glossary#input-ids)
-        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
-            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
-
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
-
-            [What are attention masks?](../glossary#attention-mask)
-
-            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
-            [`PreTrainedTokenizer.__call__`] for details.
-
-            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
-            `past_key_values`).
-
-            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
-            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
-            information on the default strategy.
-
-            - 1 indicates the head is **not masked**,
-            - 0 indicates the head is **masked**.
-        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
-            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
-            config.n_positions - 1]`.
-
-            [What are position IDs?](../glossary#position-ids)
-        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
-            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
-            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
-            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
-
-            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
-            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
-
-            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
-            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
-            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
-        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
-            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
-            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
-            model's internal embedding lookup matrix.
-        use_cache (`bool`, *optional*):
-            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
-            `past_key_values`).
-        output_attentions (`bool`, *optional*):
-            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
-            tensors for more detail.
-        output_hidden_states (`bool`, *optional*):
-            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
-            more detail.
-        return_dict (`bool`, *optional*):
-            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
-"""
-
-
-@add_start_docstrings(
-    "The bare LLaMA Model outputting raw hidden-states without any specific head on top.",
-    LLAMA_START_DOCSTRING,
-)
-class LlamaModel(LlamaPreTrainedModel):
+class InternLMModel(InternLMPreTrainedModel):
     """
-    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`]
+    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLMDecoderLayer`]
 
     Args:
-        config: LlamaConfig
+        config: InternLMXComposerConfig
     """
-    def __init__(self, config: LlamaConfig):
+    def __init__(self, config: InternLMXComposerConfig):
         super().__init__(config)
         self.padding_idx = config.pad_token_id
         self.vocab_size = config.vocab_size
@@ -934,9 +838,11 @@ class LlamaModel(LlamaPreTrainedModel):
         self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size,
                                          self.padding_idx)
         self.layers = nn.ModuleList([
-            LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)
+            InternLMDecoderLayer(config)
+            for _ in range(config.num_hidden_layers)
         ])
-        self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.norm = InternLMRMSNorm(config.hidden_size,
+                                    eps=config.rms_norm_eps)
 
         self.gradient_checkpointing = False
         # Initialize weights and apply final processing
@@ -975,7 +881,6 @@ class LlamaModel(LlamaPreTrainedModel):
 
         return combined_attention_mask
 
-    @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
     def forward(
         self,
         input_ids: torch.LongTensor = None,
@@ -1119,19 +1024,19 @@ class LlamaModel(LlamaPreTrainedModel):
         )
 
 
-class LlamaForCausalLM(LlamaPreTrainedModel):
+class InternLMForCausalLM(InternLMPreTrainedModel):
     lora_cfg = None  # init in MiniGPT4
 
     def __init__(self, config):
         super().__init__(config)
-        # TODO: find a way to explicitly initialize Llama
+        # TODO: find a way to explicitly initialize InternLM
         setattr(config, 'lora_cfg', self.lora_cfg)
 
         if hasattr(config, 'kqvo_bias'):
             setattr(config, 'kqvo_bias', config.kqvo_bias)
         else:
             setattr(config, 'kqvo_bias', False)
-        self.model = LlamaModel(config)
+        self.model = InternLMModel(config)
 
         self.lm_head = nn.Linear(config.hidden_size,
                                  config.vocab_size,
@@ -1185,9 +1090,6 @@ class LlamaForCausalLM(LlamaPreTrainedModel):
     def get_decoder(self):
         return self.model
 
-    @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
-    @replace_return_docstrings(output_type=CausalLMOutputWithPast,
-                               config_class=_CONFIG_FOR_DOC)
     def forward(
         self,
         input_ids: torch.LongTensor = None,
@@ -1214,9 +1116,9 @@ class LlamaForCausalLM(LlamaPreTrainedModel):
         Example:
 
         ```python
-        >>> from transformers import AutoTokenizer, LlamaForCausalLM
+        >>> from transformers import AutoTokenizer, InternLMForCausalLM
 
-        >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
+        >>> model = InternLMForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
         >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
 
         >>> prompt = "Hey, are you consciours? Can you talk to me?"

modeling_vit.py

@@ -7,7 +7,7 @@ import torch.nn.functional as F
 import torch.utils.checkpoint as checkpoint
 from timm.models.layers import drop_path, to_2tuple, trunc_normal_
 
-from modeling_utils import download_cached_file
+from .modeling_utils import download_cached_file
 
 
 def _cfg(url='', **kwargs):