Duplicate from allura-forge/phi-j-6b

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,25 @@
+---
+license: apache-2.0
+datasets:
+- EleutherAI/pile
+language:
+- en
+base_model:
+- EleutherAI/gpt-j-6b
+tags:
+- gptj
+- causal-lm
+---
+
+<div style="background-color: #ff6961; padding: 10px 15px; display: flex; align-items: center; max-width: 100%; box-sizing: border-box; border-radius: 5px;">
+  <span style="color: yellow; font-size: 1.6em; margin-right: 10px; line-height: 1;">⚠️</span>
+  <span style="color: black; font-family: sans-serif; font-size: 1em;">
+    This model is lightly subtly busted in 20 different ways compared to the original. It is mostly designed for further training (that will implicitly heal it from these subtle busts). You have been warned.
+  </span>
+</div>
+
+This is a conversion of GPT-J-6b by EleutherAI into a more modern architecture that it still closely maps to (in this case, the Phi 1/1.5/2 architecture). This allows for, primarily, rope scaling, as well as for creating GGUFs (it does not currently support GPT-J's original arch.) See `convert.py` for the file used to convert the weights.
+
+Note that I was originally going to use the GPT-NeoX architecture because it felt more befitting, there appears to be [a bug](https://github.com/huggingface/transformers/pull/35610#issuecomment-3111538020) in the most recent versions of Transformers, so Phi it is!
+
+Also, the `partial_rotary_factor` is selected to be `0.5` here, despite the fact that this makes no logical sense, as even though it *should* be `0.25` (`rotary_dim / head_dim` = `64 / 256` = `0.25`), `0.25` is completely babblingly incoherent and `0.5` is basically the same as the original. Whatever.

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "PhiForCausalLM"
+  ],
+  "attention_probs_dropout_prob": 0,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "hidden_act": "gelu_new",
+  "hidden_dropout_prob": 0,
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384, 
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "phi",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "rotary_emb_base": 10000,
+  "partial_rotary_factor": 0.5,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.19.0.dev0",
+  "use_cache": true,
+  "vocab_size": 50400
+}
+

convert.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# convert_to_neox.py
+import argparse, re, torch
+from safetensors.torch import load_file, save_file
+
+def cat_if_exists(keys, state, dim=0):
+    """Helper: concatenate a list of keys if they all exist, else return None."""
+    if all(k in state for k in keys):
+        return torch.cat([state[k] for k in keys], dim=dim)
+    return None
+
+def convert(path_in: str, path_out: str, dtype: str):
+    src = load_file(path_in)
+    tgt = {}
+
+    # --- top‑level tensors ---------------------------------------------------
+    tgt["model.embed_tokens.weight"]       = src["transformer.wte.weight"].to(dtype)
+    tgt["model.final_layernorm.weight"] = src["transformer.ln_f.weight"].to(dtype)
+    tgt["model.final_layernorm.bias"]   = src["transformer.ln_f.bias"].to(dtype)
+    tgt["lm_head.weight"] = src["lm_head.weight"].to(dtype)
+    tgt["lm_head.bias"] = src["lm_head.bias"].to(dtype)
+
+    # --- per‑layer tensors ---------------------------------------------------
+    pat = re.compile(r"transformer\.h\.(\d+)\.")
+    layer_ids = sorted({int(pat.match(k).group(1)) for k in src if pat.match(k)})
+
+    for i in layer_ids:
+        p_old = f"transformer.h.{i}"
+        p_new = f"model.layers.{i}"
+
+        ## attention — fuse QKV
+        #qkv_w = cat_if_exists(
+        #    [f"{p_old}.attn.q_proj.weight",
+        #     f"{p_old}.attn.k_proj.weight",
+        #     f"{p_old}.attn.v_proj.weight"],
+        #    src)
+        #qkv_b = cat_if_exists(
+        #    [f"{p_old}.attn.q_proj.bias",
+        #     f"{p_old}.attn.k_proj.bias",
+        #     f"{p_old}.attn.v_proj.bias"],
+        #    src)
+        #
+        #tgt[f"{p_new}.attention.query_key_value.weight"] = qkv_w.to(dtype)
+        #if qkv_b is not None:
+        #    tgt[f"{p_new}.attention.query_key_value.bias"] = qkv_b.to(dtype)
+        #else:
+        #    tgt[f"{p_new}.attention.query_key_value.bias"] = torch.tensor([0] * qkv_w.shape[0]).to(dtype)
+
+        tgt[f"{p_new}.self_attn.k_proj.weight"] = src[f"{p_old}.attn.k_proj.weight"]
+        tgt[f"{p_new}.self_attn.k_proj.bias"] = torch.tensor([0] * tgt[f"{p_new}.self_attn.k_proj.weight"].shape[0])
+
+        tgt[f"{p_new}.self_attn.q_proj.weight"] = src[f"{p_old}.attn.q_proj.weight"]
+        tgt[f"{p_new}.self_attn.q_proj.bias"] = torch.tensor([0] * tgt[f"{p_new}.self_attn.q_proj.weight"].shape[0])
+
+        tgt[f"{p_new}.self_attn.v_proj.weight"] = src[f"{p_old}.attn.v_proj.weight"]
+        tgt[f"{p_new}.self_attn.v_proj.bias"] = torch.tensor([0] * tgt[f"{p_new}.self_attn.v_proj.weight"].shape[0])
+
+        tgt[f"{p_new}.self_attn.dense.weight"] = src[f"{p_old}.attn.out_proj.weight"].to(dtype)
+        tgt[f"{p_new}.self_attn.dense.bias"] = torch.tensor([0] * tgt[f"{p_new}.self_attn.dense.weight"].shape[0]).to(dtype)
+
+        # layer norms
+        tgt[f"{p_new}.input_layernorm.weight"]  = src[f"{p_old}.ln_1.weight"].to(dtype)
+        tgt[f"{p_new}.input_layernorm.bias"]    = src[f"{p_old}.ln_1.bias"].to(dtype)
+
+        # MLP
+        tgt[f"{p_new}.mlp.fc1.weight"] = src[f"{p_old}.mlp.fc_in.weight"].to(dtype)
+        tgt[f"{p_new}.mlp.fc1.bias"]   = src[f"{p_old}.mlp.fc_in.bias"].to(dtype)
+        tgt[f"{p_new}.mlp.fc2.weight"] = src[f"{p_old}.mlp.fc_out.weight"].to(dtype)
+        tgt[f"{p_new}.mlp.fc2.bias"]   = src[f"{p_old}.mlp.fc_out.bias"].to(dtype)
+
+    # ------------------------------------------------------------------------
+    save_file(tgt, path_out)
+    print(f"✓ wrote {len(tgt):,} tensors to {path_out}")
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(description="convert GPT‑2/3 style safetensors to Phi1.5 layout")
+    ap.add_argument("--in",  dest="inp",  required=True, help="source .safetensors")
+    ap.add_argument("--out", dest="outp", required=True, help="destination .safetensors")
+    ap.add_argument("--dtype", default="float16",
+                    choices=["float16","bfloat16","float32"],
+                    help="cast parameters to this dtype in the output file")
+    args = ap.parse_args()
+    convert(args.inp, args.outp, getattr(torch, args.dtype))

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ffac8bf9bf940e422949610ee79efa8f332e5171a5b2dac9705be7f132f6a7e
+size 14923363064

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "errors": "replace", "model_max_length": 2048, "special_tokens_map_file": null, "name_or_path": "gpt-j-6B", "from_slow": true, "tokenizer_class": "GPT2Tokenizer"}