Merge branch 'main' of https://huggingface.co/normalcomputing/extended-mind-mpt-7b into main

blocks.py

@@ -7,7 +7,7 @@
 from typing import Dict, Optional, Tuple
 import torch
 import torch.nn as nn
-from memorizing_transformers.mpt.attention import ATTN_CLASS_REGISTRY
+from attention import ATTN_CLASS_REGISTRY
 from llmfoundry.models.layers.norm import NORM_CLASS_REGISTRY
 
 class MPTMLP(nn.Module):

config.json

@@ -21,8 +21,8 @@
     "use_active_externalism": true
   },
   "auto_map": {
-    "AutoConfig": "mosaicml/mpt-7b--configuration_mpt.MPTConfig",
-    "AutoModelForCausalLM": "mosaicml/mpt-7b--modeling_mpt.MPTForCausalLM"
+    "AutoConfig": "configuration.ExtendedMPTConfig",
+    "AutoModelForCausalLM": "modeling_mpt.ExtendedMPTForCausalLM"
   },
   "d_model": 4096,
   "emb_pdrop": 0,

modeling_mpt.py

@@ -27,10 +27,10 @@ from llmfoundry.models.layers.custom_embedding import SharedEmbedding
 from llmfoundry.models.layers.norm import NORM_CLASS_REGISTRY
 from llmfoundry.models.utils.param_init_fns import MODEL_INIT_REGISTRY
 
-from memorizing_transformers.mpt.configuration import ExtendedMPTConfig
-from memorizing_transformers.mpt.attention import attn_bias_shape, build_attn_bias
-from memorizing_transformers.mpt.blocks import MPTBlock
-from memorizing_transformers.utils import instantiate_from_config
+from configuration import ExtendedMPTConfig
+from attention import attn_bias_shape, build_attn_bias
+from blocks import MPTBlock
+from utils import instantiate_from_config
 
 Tokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]