Download generate.py from Veirded/SLLM-0.5: direct link, hf CLI and curl.
- Browser
- Download file 6.72 kB
-
https://huggingface.co/Veirded/SLLM-0.5/resolve/main/generate.py
- Command line
-
hf download hf://Veirded/SLLM-0.5/generate.py
-
curl -L -o generate.py https://huggingface.co/Veirded/SLLM-0.5/resolve/main/generate.py
6.72 kB
| import os | |
| import re | |
| import json | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| if torch.backends.mps.is_available(): | |
| device = torch.device("mps") | |
| print("Using Apple Silicon Metal (MPS) acceleration.") | |
| elif torch.cuda.is_available(): | |
| device = torch.device("cuda") | |
| print(f"Using NVIDIA CUDA acceleration: {torch.cuda.get_device_name(0)}") | |
| else: | |
| device = torch.device("cpu") | |
| print("Using CPU inference.") | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim, eps=1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def forward(self, x): | |
| return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight | |
| class SwiGLU(nn.Module): | |
| def __init__(self, dim, hidden_dim): | |
| super().__init__() | |
| self.w1 = nn.Linear(dim, hidden_dim, bias=False) | |
| self.w2 = nn.Linear(hidden_dim, dim, bias=False) | |
| self.w3 = nn.Linear(dim, hidden_dim, bias=False) | |
| def forward(self, x): | |
| return self.w2(F.silu(self.w1(x)) * self.w3(x)) | |
| def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0): | |
| freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) | |
| t = torch.arange(end, device=freqs.device) | |
| freqs = torch.outer(t, freqs).float() | |
| return torch.polar(torch.ones_like(freqs), freqs) | |
| def apply_rotary_emb(xq, xk, freqs_cis): | |
| xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) | |
| xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) | |
| freqs_cis = freqs_cis[:xq_.shape[1]].unsqueeze(0).unsqueeze(2) | |
| xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) | |
| xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) | |
| return xq_out.type_as(xq), xk_out.type_as(xk) | |
| class RoPEAttention(nn.Module): | |
| def __init__(self, d_model=112, n_head=4, block_size=256): | |
| super().__init__() | |
| self.n_head = n_head | |
| self.head_dim = d_model // n_head | |
| self.wq = nn.Linear(d_model, d_model, bias=False) | |
| self.wk = nn.Linear(d_model, d_model, bias=False) | |
| self.wv = nn.Linear(d_model, d_model, bias=False) | |
| self.wo = nn.Linear(d_model, d_model, bias=False) | |
| self.register_buffer('bias', torch.tril(torch.ones(block_size, block_size)).view(1, 1, block_size, block_size)) | |
| def forward(self, x, freqs_cis): | |
| B, T, C = x.shape | |
| q = self.wq(x).view(B, T, self.n_head, self.head_dim) | |
| k = self.wk(x).view(B, T, self.n_head, self.head_dim) | |
| v = self.wv(x).view(B, T, self.n_head, self.head_dim) | |
| q, k = apply_rotary_emb(q, k, freqs_cis) | |
| q = q.transpose(1, 2) | |
| k = k.transpose(1, 2) | |
| v = v.transpose(1, 2) | |
| att = (q @ k.transpose(-2, -1)) * (1.0 / (self.head_dim ** 0.5)) | |
| att = att.masked_fill(self.bias[:, :, :T, :T] == 0, float('-inf')) | |
| att = F.softmax(att, dim=-1) | |
| y = att @ v | |
| y = y.transpose(1, 2).contiguous().view(B, T, C) | |
| return self.wo(y) | |
| class NovaBlock(nn.Module): | |
| def __init__(self, d_model=112, n_head=4, hidden_dim=298, block_size=256): | |
| super().__init__() | |
| self.attn_norm = RMSNorm(d_model) | |
| self.attn = RoPEAttention(d_model, n_head, block_size) | |
| self.attn_gate = nn.Linear(d_model, d_model, bias=False) | |
| self.ffn_norm = RMSNorm(d_model) | |
| self.ffn = SwiGLU(d_model, hidden_dim) | |
| self.ffn_gate = nn.Linear(d_model, d_model, bias=False) | |
| def forward(self, x, freqs_cis): | |
| x = x + torch.sigmoid(self.attn_gate(x)) * self.attn(self.attn_norm(x), freqs_cis) | |
| x = x + torch.sigmoid(self.ffn_gate(x)) * self.ffn(self.ffn_norm(x)) | |
| return x | |
| class SLLM(nn.Module): | |
| def __init__(self, vocab_size=9385, d_model=112, n_head=4, n_layer=4, block_size=256): | |
| super().__init__() | |
| self.block_size = block_size | |
| self.tok_emb = nn.Embedding(vocab_size, d_model) | |
| self.layers = nn.ModuleList([NovaBlock(d_model, n_head, int(2 * (4 * d_model) / 3), block_size) for _ in range(n_layer)]) | |
| self.norm = RMSNorm(d_model) | |
| self.highway_gain = nn.Parameter(torch.tensor(0.1)) | |
| self.lm_head = nn.Linear(d_model, vocab_size, bias=False) | |
| self.register_buffer('freqs_cis', precompute_freqs_cis(d_model // n_head, block_size)) | |
| def forward(self, idx): | |
| B, T = idx.shape | |
| x_emb = self.tok_emb(idx) | |
| freqs_cis = self.freqs_cis[:T] | |
| x = x_emb | |
| for layer in self.layers: | |
| x = layer(x, freqs_cis) | |
| x_out = self.norm(x + self.highway_gain * x_emb) | |
| return self.lm_head(x_out) | |
| dir_path = os.path.dirname(os.path.abspath(__file__)) | |
| with open(os.path.join(dir_path, "tokenizer_sllm_t.json"), "r", encoding="utf-8") as f: | |
| tok_data = json.load(f) | |
| tok2id = tok_data["tok2id"] | |
| id2tok = {int(k): v for k, v in tok_data["id2tok"].items()} | |
| end_id = tok2id["<end_of_text>"] | |
| pattern = r"(?:<end_of_text>|\w+|[^\w\s])" | |
| model = SLLM(vocab_size=len(tok2id), block_size=256).to(device) | |
| model.load_state_dict(torch.load(os.path.join(dir_path, "sllm_weights.pt"), map_location=device)) | |
| model.eval() | |
| def detokenize(tokens): | |
| out = "" | |
| for t in tokens: | |
| if t in {".", ",", "!", "?", "'", ";", ":", ")", "]"}: out = out.rstrip() + t + " " | |
| elif t in {"(", "["}: out += t | |
| else: out += t + " " | |
| return out.strip() | |
| print("\n" + "="*50) | |
| print("🤖 Connected to Veirded/SLLM-0.5 (~2.8M Parameters)") | |
| print("Type 'exit' to quit.") | |
| print("="*50) | |
| history = [] | |
| while True: | |
| try: | |
| user_msg = input("\nYou: ").strip() | |
| except (KeyboardInterrupt, EOFError): | |
| break | |
| if not user_msg or user_msg.lower() in ("quit", "exit"): | |
| break | |
| user_tokens = re.findall(pattern, user_msg) | |
| history.extend(user_tokens) | |
| ctx = history[-256:] | |
| idx = torch.tensor([[tok2id.get(t, 0) for t in ctx]], dtype=torch.long, device=device) | |
| reply_tokens = [] | |
| with torch.no_grad(): | |
| for step in range(50): | |
| logits = model(idx[:, -256:])[:, -1, :] / 0.75 | |
| if step < 8: logits[0, end_id] = -float('Inf') | |
| for past_tok in set(reply_tokens[-6:]): logits[0, tok2id.get(past_tok, 0)] /= 1.4 | |
| v, _ = torch.topk(logits, min(30, logits.size(-1))) | |
| logits[logits < v[:, [-1]]] = -float('Inf') | |
| probs = F.softmax(logits, dim=-1) | |
| next_id = torch.multinomial(probs, num_samples=1) | |
| token_str = id2tok[next_id.item()] | |
| if token_str == "<end_of_text>": break | |
| reply_tokens.append(token_str) | |
| idx = torch.cat((idx, next_id), dim=1) | |
| print(f"Bot: {detokenize(reply_tokens)}") | |
| history.extend(reply_tokens) | |