Upload 7 files

LICENSE

@@ -0,0 +1,14 @@
+1. Grant of Rights
+The Author grants you a perpetual, worldwide, royalty-free right to use, copy, modify, merge, distribute, sublicense, and commercially exploit this source code, including any directly related patents owned or licensable by the Author.
+
+2. Redistribution
+When sharing or reselling the source code (whether unmodified or modified), you must include a copy of this License and retain the Author's original copyright notice.
+
+3. Limitations & Protection
+
+Trademarks: This License does not grant any rights to use the Author's name, logo, or trademarks for marketing or commercial purposes.
+
+Patents: The patent rights granted in Section 1 will automatically terminate if you initiate legal proceedings against the Author alleging patent infringement related to this source code.
+
+4. Disclaimer
+The source code is provided on an "AS IS" basis, without warranty of any kind. The Author assumes no liability or legal responsibility for any claims, losses, or damages arising from the use of the source code. Any rights not expressly restricted in this License remain at your sole discretion.

README (1).md

@@ -0,0 +1,29 @@
+---
+license: other
+language:
+- en
+tags:
+- msh
+- ggcolab
+---
+hi guys, im lazy to write, so this was written by claude, ty
+
+This is my MSH-Tiny model before the redesign (you could say it's the original version, lol)
+
+## Files
+
+- `config.json` — architecture + hyperparameters
+- `model_weights.pt` — trained PyTorch weights
+- `tokenizer.json` — custom BPE tokenizer
+
+## How to run it
+
+Use the included notebook **`You_can_still_run_it_directly_on_your_own_machine_if_it's_too_small.ipynb`** — open it in Google Colab, run all cells.
+
+Looking for a ready-to-use format instead?
+- `.safetensors`: [`mondk/Safetensors.msh-tiny`](https://huggingface.co/mondk/Safetensors.msh-tiny)
+- `.gguf`: [`mondk/GGUF.msh-tiny`](https://huggingface.co/mondk/GGUF.msh-tiny)
+
+---
+
+my license: https://huggingface.co/mondk/see_upcoming_models/blob/main/LICENSE

You_can_still_run_it_directly_on_your_own_machine_if_it%27s_too_small.ipynb

@@ -0,0 +1,232 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OEKVU8aAXeQx"
+      },
+      "outputs": [],
+      "source": [
+        "REPO_ID = \"mondk/msh-tiny.For-Test-In-Note-Book-Google-Colab\"  # <-- CHANGE THIS to the actual Hugging Face repo, e.g. \"john/my-chat-model\"\n"
+      ],
+      "id": "OEKVU8aAXeQx"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Muc0ftk9XeQz"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q -U huggingface_hub tokenizers\n",
+        "import torch\n",
+        "print(\"PyTorch:\", torch.__version__)\n",
+        "print(\"CUDA available:\", torch.cuda.is_available())\n"
+      ],
+      "id": "Muc0ftk9XeQz"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "h5Lf-nDoXeQ0"
+      },
+      "outputs": [],
+      "source": [
+        "from huggingface_hub import hf_hub_download\n",
+        "\n",
+        "local_dir = \"/content/downloaded_model\"\n",
+        "config_path = hf_hub_download(repo_id=REPO_ID, filename=\"config.json\", local_dir=local_dir)\n",
+        "weights_path = hf_hub_download(repo_id=REPO_ID, filename=\"model_weights.pt\", local_dir=local_dir)\n",
+        "tokenizer_path = hf_hub_download(repo_id=REPO_ID, filename=\"tokenizer.json\", local_dir=local_dir)\n",
+        "\n",
+        "print(\"Downloaded to:\", local_dir)\n"
+      ],
+      "id": "h5Lf-nDoXeQ0"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rhTG3mtsXeQ0"
+      },
+      "outputs": [],
+      "source": [
+        "import math\n",
+        "import torch.nn as nn\n",
+        "import torch.nn.functional as F\n",
+        "\n",
+        "class CausalSelfAttention(nn.Module):\n",
+        "    def __init__(self, n_embd, n_head, block_size, dropout=0.1):\n",
+        "        super().__init__()\n",
+        "        assert n_embd % n_head == 0\n",
+        "        self.n_head = n_head\n",
+        "        self.head_dim = n_embd // n_head\n",
+        "        self.qkv = nn.Linear(n_embd, 3 * n_embd)\n",
+        "        self.proj = nn.Linear(n_embd, n_embd)\n",
+        "        self.attn_dropout = nn.Dropout(dropout)\n",
+        "        self.resid_dropout = nn.Dropout(dropout)\n",
+        "        mask = torch.tril(torch.ones(block_size, block_size)).view(1, 1, block_size, block_size)\n",
+        "        self.register_buffer(\"mask\", mask)\n",
+        "\n",
+        "    def forward(self, x):\n",
+        "        B, T, C = x.shape\n",
+        "        qkv = self.qkv(x)\n",
+        "        q, k, v = qkv.split(C, dim=2)\n",
+        "        q = q.view(B, T, self.n_head, self.head_dim).transpose(1, 2)\n",
+        "        k = k.view(B, T, self.n_head, self.head_dim).transpose(1, 2)\n",
+        "        v = v.view(B, T, self.n_head, self.head_dim).transpose(1, 2)\n",
+        "\n",
+        "        att = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)\n",
+        "        att = att.masked_fill(self.mask[:, :, :T, :T] == 0, float(\"-inf\"))\n",
+        "        att = F.softmax(att, dim=-1)\n",
+        "        att = self.attn_dropout(att)\n",
+        "        out = att @ v\n",
+        "        out = out.transpose(1, 2).contiguous().view(B, T, C)\n",
+        "        return self.resid_dropout(self.proj(out))\n",
+        "\n",
+        "\n",
+        "class MLP(nn.Module):\n",
+        "    def __init__(self, n_embd, dropout=0.1):\n",
+        "        super().__init__()\n",
+        "        self.fc1 = nn.Linear(n_embd, 4 * n_embd)\n",
+        "        self.fc2 = nn.Linear(4 * n_embd, n_embd)\n",
+        "        self.dropout = nn.Dropout(dropout)\n",
+        "\n",
+        "    def forward(self, x):\n",
+        "        return self.dropout(self.fc2(F.gelu(self.fc1(x))))\n",
+        "\n",
+        "\n",
+        "class Block(nn.Module):\n",
+        "    def __init__(self, n_embd, n_head, block_size, dropout=0.1):\n",
+        "        super().__init__()\n",
+        "        self.ln1 = nn.LayerNorm(n_embd)\n",
+        "        self.attn = CausalSelfAttention(n_embd, n_head, block_size, dropout)\n",
+        "        self.ln2 = nn.LayerNorm(n_embd)\n",
+        "        self.mlp = MLP(n_embd, dropout)\n",
+        "\n",
+        "    def forward(self, x):\n",
+        "        x = x + self.attn(self.ln1(x))\n",
+        "        x = x + self.mlp(self.ln2(x))\n",
+        "        return x\n",
+        "\n",
+        "\n",
+        "class GPTFromScratch(nn.Module):\n",
+        "    def __init__(self, vocab_size, block_size, n_layer=6, n_head=6, n_embd=384, dropout=0.1):\n",
+        "        super().__init__()\n",
+        "        self.block_size = block_size\n",
+        "        self.tok_emb = nn.Embedding(vocab_size, n_embd)\n",
+        "        self.pos_emb = nn.Embedding(block_size, n_embd)\n",
+        "        self.drop = nn.Dropout(dropout)\n",
+        "        self.blocks = nn.ModuleList([Block(n_embd, n_head, block_size, dropout) for _ in range(n_layer)])\n",
+        "        self.ln_f = nn.LayerNorm(n_embd)\n",
+        "        self.head = nn.Linear(n_embd, vocab_size, bias=False)\n",
+        "        self.head.weight = self.tok_emb.weight  # weight tying\n",
+        "\n",
+        "    def forward(self, idx, targets=None):\n",
+        "        B, T = idx.shape\n",
+        "        pos = torch.arange(T, device=idx.device).unsqueeze(0)\n",
+        "        x = self.drop(self.tok_emb(idx) + self.pos_emb(pos))\n",
+        "        for block in self.blocks:\n",
+        "            x = block(x)\n",
+        "        x = self.ln_f(x)\n",
+        "        logits = self.head(x)\n",
+        "\n",
+        "        loss = None\n",
+        "        if targets is not None:\n",
+        "            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))\n",
+        "        return logits, loss\n",
+        "\n",
+        "    @torch.no_grad()\n",
+        "    def generate(self, idx, max_new_tokens, temperature=0.8, top_k=40):\n",
+        "        for _ in range(max_new_tokens):\n",
+        "            idx_cond = idx[:, -self.block_size :]\n",
+        "            logits, _ = self(idx_cond)\n",
+        "            logits = logits[:, -1, :] / temperature\n",
+        "            if top_k is not None:\n",
+        "                v, _ = torch.topk(logits, top_k)\n",
+        "                logits[logits < v[:, [-1]]] = float(\"-inf\")\n",
+        "            probs = F.softmax(logits, dim=-1)\n",
+        "            next_id = torch.multinomial(probs, num_samples=1)\n",
+        "            idx = torch.cat([idx, next_id], dim=1)\n",
+        "        return idx\n"
+      ],
+      "id": "rhTG3mtsXeQ0"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PP6KcuhQXeQ1"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "\n",
+        "from tokenizers import Tokenizer\n",
+        "\n",
+        "with open(config_path) as f:\n",
+        "    cfg = json.load(f)\n",
+        "\n",
+        "tokenizer_backend = Tokenizer.from_file(tokenizer_path)\n",
+        "\n",
+        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+        "model = GPTFromScratch(\n",
+        "    vocab_size=cfg[\"vocab_size\"],\n",
+        "    block_size=cfg[\"block_size\"],\n",
+        "    n_layer=cfg[\"n_layer\"],\n",
+        "    n_head=cfg[\"n_head\"],\n",
+        "    n_embd=cfg[\"n_embd\"],\n",
+        ").to(device)\n",
+        "model.load_state_dict(torch.load(weights_path, map_location=device))\n",
+        "model.eval()\n",
+        "\n",
+        "USER_TOK = cfg[\"user_tok\"]\n",
+        "ASSISTANT_TOK = cfg[\"assistant_tok\"]\n",
+        "END_TOK = cfg[\"end_tok\"]\n",
+        "END_ID = tokenizer_backend.token_to_id(END_TOK)\n",
+        "\n",
+        "n_params = sum(p.numel() for p in model.parameters())\n",
+        "print(f\"Model loaded successfully: {n_params:,} parameters, running on {device}\")\n"
+      ],
+      "id": "PP6KcuhQXeQ1"
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "u1CRQJL2XeQ2"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Chat started. Type 'exit' to quit.\\n\")\n",
+        "while True:\n",
+        "    user_input = input(\"You: \")\n",
+        "    if user_input.strip().lower() == \"exit\":\n",
+        "        print(\"Goodbye!\")\n",
+        "        break\n",
+        "    reply = chat(user_input)\n",
+        "    print(\"Bot:\", reply)\n"
+      ],
+      "id": "u1CRQJL2XeQ2"
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 5
+}

config.json

@@ -0,0 +1,10 @@
+{
+  "vocab_size": 8192,
+  "block_size": 256,
+  "n_layer": 6,
+  "n_head": 6,
+  "n_embd": 384,
+  "user_tok": "<|user|>",
+  "assistant_tok": "<|assistant|>",
+  "end_tok": "<|end|>"
+}

gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

model_weights.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70769a57a51f4ebf1526ecbf57ebc437b40f28a323a3a3cb1238c5810d9f4f40
+size 57169021