Download generate.py from Veirded/SLLM-0.6: direct link, hf CLI and curl.
- Browser
- Download file 9.22 kB
-
https://huggingface.co/Veirded/SLLM-0.6/resolve/main/generate.py
- Command line
-
hf download hf://Veirded/SLLM-0.6/generate.py
-
curl -L -o generate.py https://huggingface.co/Veirded/SLLM-0.6/resolve/main/generate.py
9.22 kB
| import re | |
| import json | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from safetensors.torch import load_model | |
| if torch.backends.mps.is_available(): | |
| device = torch.device("mps") | |
| print("Using Apple Silicon Metal (MPS) acceleration.") | |
| elif torch.cuda.is_available(): | |
| device = torch.device("cuda") | |
| print(f"Using NVIDIA CUDA acceleration: {torch.cuda.get_device_name(0)}") | |
| else: | |
| device = torch.device("cpu") | |
| print("Using CPU inference.") | |
| SPECIAL_TOKENS = ["<pad>", "<end_of_text>", "<|user|>", "<|ai|>"] | |
| PRETOK_RE = re.compile( | |
| r"(?i:'(?:s|t|re|ve|m|ll|d))| ?[A-Za-z]+| ?[0-9]+| ?[^\sA-Za-z0-9]+|\s+(?!\S)|\s+" | |
| ) | |
| SPECIAL_RE = re.compile("(" + "|".join(re.escape(tok) for tok in SPECIAL_TOKENS) + ")") | |
| class ByteBPETokenizer: | |
| def __init__(self, special_tokens, merges): | |
| self.special_tokens = list(special_tokens) | |
| self.merges = [tuple(m) for m in merges] | |
| self.tok2id = {tok: i for i, tok in enumerate(self.special_tokens)} | |
| self.pad_id = self.tok2id["<pad>"] | |
| self.end_id = self.tok2id["<end_of_text>"] | |
| self.user_id = self.tok2id["<|user|>"] | |
| self.ai_id = self.tok2id["<|ai|>"] | |
| self.byte_offset = len(self.special_tokens) | |
| self.id_to_bytes = {i: b"" for i in range(len(self.special_tokens))} | |
| for b in range(256): | |
| self.id_to_bytes[self.byte_offset + b] = bytes([b]) | |
| self.merge_ranks = {} | |
| for rank, (a, b, new_id) in enumerate(self.merges): | |
| self.id_to_bytes[new_id] = self.id_to_bytes[a] + self.id_to_bytes[b] | |
| self.merge_ranks[(a, b)] = (rank, new_id) | |
| def load(cls, path): | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| return cls(data["special_tokens"], data["merges"]) | |
| def _encode_word(self, word_bytes): | |
| ids = [self.byte_offset + b for b in word_bytes] | |
| while len(ids) >= 2: | |
| best_rank, best_pair, best_new_id = None, None, None | |
| for i in range(len(ids) - 1): | |
| info = self.merge_ranks.get((ids[i], ids[i + 1])) | |
| if info is not None and (best_rank is None or info[0] < best_rank): | |
| best_rank, best_new_id = info | |
| best_pair = (ids[i], ids[i + 1]) | |
| if best_pair is None: | |
| break | |
| a, b = best_pair | |
| merged, i, n = [], 0, len(ids) | |
| while i < n: | |
| if i + 1 < n and ids[i] == a and ids[i + 1] == b: | |
| merged.append(best_new_id) | |
| i += 2 | |
| else: | |
| merged.append(ids[i]) | |
| i += 1 | |
| ids = merged | |
| return ids | |
| def encode(self, text): | |
| ids = [] | |
| for part in SPECIAL_RE.split(text): | |
| if not part: | |
| continue | |
| sid = self.tok2id.get(part) | |
| if sid is not None: | |
| ids.append(sid) | |
| else: | |
| for chunk in PRETOK_RE.findall(part): | |
| ids.extend(self._encode_word(chunk.encode("utf-8"))) | |
| return ids | |
| def decode(self, ids): | |
| special_set = set(range(len(self.special_tokens))) | |
| chunks = [self.id_to_bytes[int(i)] for i in ids if int(i) not in special_set and int(i) in self.id_to_bytes] | |
| return b"".join(chunks).decode("utf-8", errors="replace") | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim, eps=1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def forward(self, x): | |
| return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight | |
| class SwiGLU(nn.Module): | |
| def __init__(self, dim, hidden_dim): | |
| super().__init__() | |
| self.w1 = nn.Linear(dim, hidden_dim, bias=False) | |
| self.w2 = nn.Linear(hidden_dim, dim, bias=False) | |
| self.w3 = nn.Linear(dim, hidden_dim, bias=False) | |
| def forward(self, x): | |
| return self.w2(F.silu(self.w1(x)) * self.w3(x)) | |
| def precompute_rope_tables(dim, max_seq_len, theta=10000.0): | |
| inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim)) | |
| freqs = torch.outer(torch.arange(max_seq_len, dtype=torch.float32), inv_freq) | |
| emb = torch.cat((freqs, freqs), dim=-1) | |
| return emb.cos(), emb.sin() | |
| def rotate_half(x): | |
| half = x.shape[-1] // 2 | |
| return torch.cat((-x[..., half:], x[..., :half]), dim=-1) | |
| class RoPEAttention(nn.Module): | |
| def __init__(self, d_model, n_head): | |
| super().__init__() | |
| self.n_head = n_head | |
| self.head_dim = d_model // n_head | |
| self.wq = nn.Linear(d_model, d_model, bias=False) | |
| self.wk = nn.Linear(d_model, d_model, bias=False) | |
| self.wv = nn.Linear(d_model, d_model, bias=False) | |
| self.wo = nn.Linear(d_model, d_model, bias=False) | |
| self.q_norm = RMSNorm(self.head_dim) | |
| self.k_norm = RMSNorm(self.head_dim) | |
| def forward(self, x, cos, sin, attn_mask): | |
| B, T, C = x.shape | |
| q = self.q_norm(self.wq(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)) | |
| k = self.k_norm(self.wk(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)) | |
| v = self.wv(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2) | |
| q = (q * cos) + (rotate_half(q) * sin) | |
| k = (k * cos) + (rotate_half(k) * sin) | |
| att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(self.head_dim)) | |
| att = F.softmax(att.masked_fill(~attn_mask, float("-inf")).float(), dim=-1).type_as(v) | |
| return self.wo((att @ v).transpose(1, 2).contiguous().view(B, T, C)) | |
| class SLLMBlock(nn.Module): | |
| def __init__(self, d_model, n_head, hidden_dim): | |
| super().__init__() | |
| self.attn_norm = RMSNorm(d_model) | |
| self.attn = RoPEAttention(d_model, n_head) | |
| self.ffn_norm = RMSNorm(d_model) | |
| self.ffn = SwiGLU(d_model, hidden_dim) | |
| def forward(self, x, cos, sin, attn_mask): | |
| x = x + self.attn(self.attn_norm(x), cos, sin, attn_mask) | |
| return x + self.ffn(self.ffn_norm(x)) | |
| class SLLM(nn.Module): | |
| def __init__(self, vocab_size, d_model, n_head, n_layer, hidden_dim, block_size, pad_id=0): | |
| super().__init__() | |
| self.block_size = block_size | |
| self.pad_id = pad_id | |
| self.tok_emb = nn.Embedding(vocab_size, d_model) | |
| self.layers = nn.ModuleList([SLLMBlock(d_model, n_head, hidden_dim) for _ in range(n_layer)]) | |
| self.norm = RMSNorm(d_model) | |
| self.lm_head = nn.Linear(d_model, vocab_size, bias=False) | |
| self.lm_head.weight = self.tok_emb.weight | |
| cos, sin = precompute_rope_tables(d_model // n_head, block_size) | |
| self.register_buffer("cos", cos, persistent=False) | |
| self.register_buffer("sin", sin, persistent=False) | |
| self.register_buffer("causal_mask", torch.tril(torch.ones(block_size, block_size, dtype=torch.bool)).view(1, 1, block_size, block_size), persistent=False) | |
| self.register_buffer("eye_mask", torch.eye(block_size, dtype=torch.bool).view(1, 1, block_size, block_size), persistent=False) | |
| def forward(self, idx): | |
| B, T = idx.shape | |
| non_pad_k = (idx != self.pad_id).unsqueeze(1).unsqueeze(2) | |
| pad_q_diag = (idx == self.pad_id).unsqueeze(1).unsqueeze(3) & self.eye_mask[:, :, :T, :T] | |
| attn_mask = self.causal_mask[:, :, :T, :T] & (non_pad_k | pad_q_diag) | |
| x = self.tok_emb(idx) | |
| cos, sin = self.cos[:T].unsqueeze(0).unsqueeze(0), self.sin[:T].unsqueeze(0).unsqueeze(0) | |
| for layer in self.layers: | |
| x = layer(x, cos, sin, attn_mask) | |
| return self.lm_head(self.norm(x)) | |
| with open("config.json", "r") as f: | |
| cfg = json.load(f) | |
| tokenizer = ByteBPETokenizer.load("tokenizer.json") | |
| model = SLLM( | |
| vocab_size=cfg["vocab_size"], | |
| d_model=cfg["d_model"], | |
| n_head=cfg["n_head"], | |
| n_layer=cfg["n_layer"], | |
| hidden_dim=cfg["hidden_dim"], | |
| block_size=cfg["block_size"], | |
| pad_id=tokenizer.pad_id, | |
| ).to(device) | |
| load_model(model, "model.safetensors") | |
| model.eval() | |
| print("\n" + "=" * 50) | |
| print("Connected to Veirded/SLLM-0.6 (1.0M Parameters)") | |
| print("Single-turn mode. Type 'quit' to exit.") | |
| print("=" * 50) | |
| while True: | |
| uin = input("\nYou: ").strip() | |
| if not uin or uin.lower() in ("quit", "exit"): | |
| break | |
| ids = tokenizer.encode(f"<|user|> {uin}\n<|ai|>")[-cfg["block_size"]:] | |
| idx = torch.tensor([ids], dtype=torch.long, device=device) | |
| gen = [] | |
| with torch.no_grad(): | |
| for _ in range(64): | |
| logits = model(idx[:, -cfg["block_size"]:])[0, -1].float() | |
| logits[tokenizer.pad_id] = -float("inf") | |
| for t in set(idx[0, -64:].tolist()): | |
| logits[t] = logits[t] * 1.15 if logits[t] < 0 else logits[t] / 1.15 | |
| logits = logits / 0.5 | |
| v, _ = torch.topk(logits, 15) | |
| logits[logits < v[-1]] = -float("inf") | |
| nid = int(torch.multinomial(F.softmax(logits, dim=-1), 1).item()) | |
| if nid in (tokenizer.end_id, tokenizer.user_id): | |
| break | |
| gen.append(nid) | |
| idx = torch.cat((idx, torch.tensor([[nid]], dtype=torch.long, device=device)), dim=1) | |
| print(f"SLLM: {tokenizer.decode(gen).strip()}") | |