Upload model

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "EleutherAI/pythia-160m",
+  "architectures": [
+    "CustomModel4"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoModelForCausalLM": "modeling_custom4.CustomModel4"
+  },
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "rope_scaling": null,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.31.0",
+  "use_cache": true,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

modeling_custom4.py

@@ -0,0 +1,49 @@
+# https://huggingface.co/docs/transformers/custom_models
+
+from transformers import PreTrainedModel, GPTNeoXForCausalLM, AutoModelForCausalLM, AutoTokenizer, LlamaConfig
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from torch.nn.functional import log_softmax
+from torch.nn.modules.container import ModuleList
+
+class CustomModel4(PreTrainedModel):
+    config_class = LlamaConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+
+    def forward(self, *args, labels=None, **kwargs):
+        loss = None
+        logits = None
+        for model, coeff in zip(self.models, self.coeffs):
+            logp = log_softmax(model.forward(*args, **kwargs).logits, dim=-1)
+            logits = coeff * logp if logits is None else logits + coeff * logp
+        # The rest copied from modeling_llama.py:
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        return CausalLMOutputWithPast(loss=loss, logits=logits)
+        
+
+    @classmethod
+    def combine_models(cls, *args, coeffs = [], **kwargs):
+        models = []
+        for model in args:
+            models.append(AutoModelForCausalLM.from_pretrained(model, **kwargs).eval())
+        if coeffs == []:
+            coeffs = [1/len(args)] * len(args)
+        m = cls(models[0].config)
+        m.models = ModuleList(models)
+        m.coeffs = coeffs
+        return m
+
+
+CustomModel4.register_for_auto_class('AutoModelForCausalLM')

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ffa03b589263eccf2e09157196fab7b2abdaece84c8ed0f4b18f06540f48fd0
+size 465579541